diff options
| author | nga <nga@yandex-team.ru> | 2022-02-10 16:48:09 +0300 |
|---|---|---|
| committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:48:09 +0300 |
| commit | c2a1af049e9deca890e9923abe64fe6c59060348 (patch) | |
| tree | b222e5ac2e2e98872661c51ccceee5da0d291e13 | |
| parent | 1f553f46fb4f3c5eec631352cdd900a0709016af (diff) | |
| download | ydb-c2a1af049e9deca890e9923abe64fe6c59060348.tar.gz | |
Restoring authorship annotation for <nga@yandex-team.ru>. Commit 2 of 2.
543 files changed, 24906 insertions, 24906 deletions
diff --git a/contrib/libs/sparsehash/AUTHORS b/contrib/libs/sparsehash/AUTHORS index 5bd5a287f9..d8c24c64ca 100644 --- a/contrib/libs/sparsehash/AUTHORS +++ b/contrib/libs/sparsehash/AUTHORS @@ -1,2 +1,2 @@ -google-sparsehash@googlegroups.com - +google-sparsehash@googlegroups.com + diff --git a/contrib/libs/sparsehash/COPYING b/contrib/libs/sparsehash/COPYING index ef86b820ed..e4956cfd9f 100644 --- a/contrib/libs/sparsehash/COPYING +++ b/contrib/libs/sparsehash/COPYING @@ -1,28 +1,28 @@ -Copyright (c) 2005, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +Copyright (c) 2005, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/libs/sparsehash/ChangeLog b/contrib/libs/sparsehash/ChangeLog index ce24746e9a..fd53c6f8fd 100644 --- a/contrib/libs/sparsehash/ChangeLog +++ b/contrib/libs/sparsehash/ChangeLog @@ -3,277 +3,277 @@ Mon Oct 12 21:00:00 2015 Google Inc. <google-sparsehash@googlegroups.com> * sparsehash: version 2.0.3 * Fix compilation on modern compilers and operating systems -Thu Feb 23 23:47:18 2012 Google Inc. <google-sparsehash@googlegroups.com> - - * sparsehash: version 2.0.2 - * BUGFIX: Fix backwards compatibility for <google> include folders - -Wed Feb 01 02:57:48 2012 Google Inc. <google-sparsehash@googlegroups.com> - - * sparsehash: version 2.0.1 - * BUGFIX: Fix path to malloc_extension.h in time_hash_map.cc - -Tue Jan 31 11:33:04 2012 Google Inc. <google-sparsehash@googlegroups.com> - - * sparsehash: version 2.0 - * Renamed include directory from google/ to sparsehash/ (csilvers) - * Changed the 'official' sparsehash email in setup.py/etc - * Renamed google-sparsehash.sln to sparsehash.sln - * Changed copyright text to reflect Google's relinquished ownership - -Tue Dec 20 21:04:04 2011 Google Inc. <opensource@google.com> - - * sparsehash: version 1.12 release - * Add support for serializing/unserializing dense_hash_map/set to disk - * New simpler and more flexible serialization API - * Be more consistent about clearing on unserialize() even if it fails - * Quiet some compiler warnings about unused variables - * Add a timing test for iterating (suggested by google code issue 77) - * Add offset_to_pos, the opposite of pos_to_offset, to sparsetable - * PORTING: Add some missing #includes, needed on some systems - * Die at configure-time when g++ isn't installed - * Successfully make rpm's even when dpkg is missing - * Improve deleted key test in util/gtl/{dense,sparse}hashtable - * Update automake to 1.10.1, and autoconf to 2.62 - -Thu Jun 23 21:12:58 2011 Google Inc. <opensource@google.com> - - * sparsehash: version 1.11 release - * Improve performance on pointer keys by ignoring always-0 low bits - * Fix missing $(top_srcdir) in Makefile.am, which broke some compiles - * BUGFIX: Fix a crashing typo-bug in swap() - * PORTING: Remove support for old compilers that do not use 'std' - * Add some new benchmarks to test for a place dense_hash_* does badly - * Some cosmetic changes due to a switch to a new releasing tool - -Thu Jan 20 16:07:39 2011 Google Inc. <opensource@google.com> - - * sparsehash: version 1.10 release - * Follow ExtractKey return type, allowing it to return a reference - * PORTING: fix MSVC 10 warnings (constifying result_type, placement-new) - * Update from autoconf 2.61 to autoconf 2.65 - -Fri Sep 24 11:37:50 2010 Google Inc. <opensource@google.com> - - * sparsehash: version 1.9 release - * Add is_enum; make all enums PODs by default (romanp) - * Make find_or_insert() usable directly (dawidk) - * Use zero-memory trick for allocators to reduce space use (guilin) - * Fix some compiler warnings (chandlerc, eraman) - * BUGFIX: int -> size_type in one function we missed (csilvers) - * Added sparsehash.pc, for pkg-config (csilvers) - -Thu Jul 29 15:01:29 2010 Google Inc. <opensource@google.com> - - * sparsehash: version 1.8.1 release - * Remove -Werror from Makefile: gcc 4.3 gives spurious warnings - -Thu Jul 29 09:53:26 2010 Google Inc. <opensource@google.com> - - * sparsehash: version 1.8 release - * More support for Allocator, including allocator ctor arg (csilvers) - * Repack hasthable vars to reduce container size *more* (giao) - * Speed up clear() (csilvers) - * Change HT_{OCCUPANCY,SHRINK}_FLT from float to int (csilvers) - * Revamp test suite for more complete code & timing coverage (csilvers) - * BUGFIX: Enforce max_size for dense/sparse_hashtable (giao, csilvers) - * BUGFIX: Raise exception instead of crashing on overflow (csilvers) - * BUGFIX: Allow extraneous const in key type (csilvers) - * BUGFIX: Allow same functor for both hasher and key_equals (giao) - * PORTING: remove is_convertible, which gives AIX cc fits (csilvers) - * PORTING: Renamed README.windows to README_windows.txt (csilvers) - * Created non-empty NEWS file (csilvers) - -Wed Mar 31 12:32:03 2010 Google Inc. <opensource@google.com> - - * sparsehash: version 1.7 release - * Add support for Allocator (guilin) - * Add libc_allocator_with_realloc as the new default allocator (guilin) - * Repack {sparse,dense}hashtable vars to reduce container size (giao) - * BUGFIX: operator== no longer requires same table ordering (csilvers) - * BUGFIX: fix dense_hash_*(it,it) by requiring empty-key too (csilvers) - * PORTING: fix language bugs that gcc allowed (csilvers, chandlerc) - * Update from autoconf 2.61 to autoconf 2.64 - -Fri Jan 8 14:47:55 2010 Google Inc. <opensource@google.com> - - * sparsehash: version 1.6 release - * New accessor methods for deleted_key, empty_key (sjackman) - * Use explicit hash functions in sparsehash tests (csilvers) - * BUGFIX: Cast resize to fix SUNWspro bug (csilvers) - * Check for sz overflow in min_size (csilvers) - * Speed up clear() for dense and sparse hashtables (jeff) - * Avoid shrinking in all cases when min-load is 0 (shaunj, csilvers) - * Improve densehashtable code for the deleted key (gpike) - * BUGFIX: Fix operator= when the 2 empty-keys differ (andreidam) - * BUGFIX: Fix ht copying when empty-key isn't set (andreidam) - * PORTING: Use TmpFile() instead of /tmp on MinGW (csilvers) - * PORTING: Use filenames that work with Stratus VOS. - -Tue May 12 14:16:38 2009 Google Inc. <opensource@google.com> - - * sparsehash: version 1.5.2 release - * Fix compile error: not initializing set_key in all constructors - -Fri May 8 15:23:44 2009 Google Inc. <opensource@google.com> - - * sparsehash: version 1.5.1 release - * Fix broken equal_range() for all the hash-classes (csilvers) - -Wed May 6 11:28:49 2009 Google Inc. <opensource@google.com> - - * sparsehash: version 1.5 release - * Support the tr1 unordered_map (and unordered_set) API (csilvers) - * Store only key for delkey; reduces need for 0-arg c-tor (csilvers) - * Prefer unordered_map to hash_map for the timing test (csilvers) - * PORTING: update the resource use for 64-bit machines (csilvers) - * PORTING: fix MIN/MAX collisions by un-#including windows.h (csilvers) - * Updated autoconf version to 2.61 and libtool version to 1.5.26 - -Wed Jan 28 17:11:31 2009 Google Inc. <opensource@google.com> - - * sparsehash: version 1.4 release - * Allow hashtables to be <32 buckets (csilvers) - * Fix initial-sizing bug: was sizing tables too small (csilvers) - * Add asserts that clients don't abuse deleted/empty key (csilvers) - * Improve determination of 32/64 bit for C code (csilvers) - * Small fix for doc files in rpm (csilvers) - -Thu Nov 6 15:06:09 2008 Google Inc. <opensource@google.com> - - * sparsehash: version 1.3 release - * Add an interface to change the parameters for resizing (myl) - * Document another potentially good hash function (csilvers) - -Thu Sep 18 13:53:20 2008 Google Inc. <opensource@google.com> - - * sparsehash: version 1.2 release - * Augment documentation to better describe namespace issues (csilvers) - * BUG FIX: replace hash<> with SPARSEHASH_HASH, for windows (csilvers) - * Add timing test to unittest to test repeated add+delete (csilvers) - * Do better picking a new size when resizing (csilvers) - * Use ::google instead of google as a namespace (csilvers) - * Improve threading test at config time (csilvers) - -Mon Feb 11 16:30:11 2008 Google Inc. <opensource@google.com> - - * sparsehash: version 1.1 release - * Fix brown-paper-bag bug in some constructors (rafferty) - * Fix problem with variables shadowing member vars, add -Wshadow - -Thu Nov 29 11:44:38 2007 Google Inc. <opensource@google.com> - - * sparsehash: version 1.0.2 release - * Fix a final reference to hash<> to use SPARSEHASH_HASH<> instead. - -Wed Nov 14 08:47:48 2007 Google Inc. <opensource@google.com> - - * sparsehash: version 1.0.1 release :-( - * Remove an unnecessary (harmful) "#define hash" in windows' config.h - -Tue Nov 13 15:15:46 2007 Google Inc. <opensource@google.com> - - * sparsehash: version 1.0 release! We are now out of beta. - * Clean up Makefile awk script to be more readable (csilvers) - * Namespace fixes: use fewer #defines, move typedefs into namespace - -Fri Oct 12 12:35:24 2007 Google Inc. <opensource@google.com> - - * sparsehash: version 0.9.1 release - * Fix Makefile awk script to work on more architectures (csilvers) - * Add test to test code in more 'real life' situations (csilvers) - -Tue Oct 9 14:15:21 2007 Google Inc. <opensource@google.com> - - * sparsehash: version 0.9 release - * More type-hygiene improvements, especially for 64-bit (csilvers) - * Some configure improvements to improve portability, utility (austern) - * Small bugfix for operator== for dense_hash_map (jeff) - -Tue Jul 3 12:55:04 2007 Google Inc. <opensource@google.com> - - * sparsehash: version 0.8 release - * Minor type-hygiene improvements: size_t for int, etc. (csilvers) - * Porting improvements: tests pass on OS X, FreeBSD, Solaris (csilvers) - * Full windows port! VS solution provided for all unittests (csilvers) - -Mon Jun 11 11:33:41 2007 Google Inc. <opensource@google.com> - - * sparsehash: version 0.7 release - * Syntax fixes to better support gcc 4.3 and VC++ 7 (mec, csilvers) - * Improved windows/VC++ support (see README.windows) (csilvers) - * Config improvements: better tcmalloc support and config.h (csilvers) - * More robust with missing hash_map + nix 'trampoline' .h's (csilvers) - * Support for STLport's hash_map/hash_fun locations (csilvers) - * Add .m4 files to distribution; now all source is there (csilvers) - * Tiny modification of shrink-threshhold to allow never-shrinking (amc) - * Protect timing tests against aggressive optimizers (csilvers) - * Extend time_hash_map to test bigger objects (csilvers) - * Extend type-trait support to work with const objects (csilvers) - * USER VISIBLE: speed up all code by replacing memmove with memcpy - (csilvers) - -Tue Mar 20 17:29:34 2007 Google Inc. <opensource@google.com> - - * sparsehash: version 0.6 release - * Some improvement to type-traits (jyasskin) - * Better timing results when google-perftools is installed (sanjay) - * Updates and fixes to html documentation and README (csilvers) - * A bit more careful about #includes (csilvers) - * Fix for typo that broken compilation on some systems (csilvers) - * USER VISIBLE: New clear_no_resize() method added to dense_hash_map - (uszkoreit) - -Sat Oct 21 13:47:47 2006 Google Inc. <opensource@google.com> - - * sparsehash: version 0.5 release - * Support uint16_t (SunOS) in addition to u_int16_t (BSD) (csilvers) - * Get rid of UNDERSTANDS_ITERATOR_TAGS; everyone understands (csilvers) - * Test that empty-key and deleted-key differ (rbayardo) - * Fix example docs: strcmp needs to test for NULL (csilvers) - -Sun Apr 23 22:42:35 2006 Google Inc. <opensource@google.com> - - * sparsehash: version 0.4 release - * Remove POD requirement for keys and values! (austern) - * Add tr1-compatible type-traits system to speed up POD ops. (austern) - * Fixed const-iterator bug where postfix ++ didn't compile. (csilvers) - * Fixed iterator comparison bugs where <= was incorrect. (csilvers) - * Clean up config.h to keep its #defines from conflicting. (csilvers) - * Big documentation sweep and cleanup. (csilvers) - * Update documentation to talk more about good hash fns. (csilvers) - * Fixes to compile on MSVC (working around some MSVC bugs). (rennie) - * Avoid resizing hashtable on operator[] lookups (austern) - -Thu Nov 3 20:12:31 2005 Google Inc. <opensource@google.com> - - * sparsehash: version 0.3 release - * Quiet compiler warnings on some compilers. (csilvers) - * Some documentation fixes: example code for dense_hash_map. (csilvers) - * Fix a bug where swap() wasn't swapping delete_key(). (csilvers) - * set_deleted_key() and set_empty_key() now take a key only, - allowing hash-map values to be forward-declared. (csilvers) - * support for std::insert_iterator (and std::inserter). (csilvers) - -Mon May 2 07:04:46 2005 Google Inc. <opensource@google.com> - - * sparsehash: version 0.2 release - * Preliminary support for msvc++ compilation. (csilvers) - * Documentation fixes -- some example code was incomplete! (csilvers) - * Minimize size of config.h to avoid other-package conflicts (csilvers) - * Contribute a C-based version of sparsehash that served as the - inspiration for this code. One day, I hope to clean it up and - support it, but for now it's just in experimental/, for playing - around with. (csilvers) - * Change default namespace from std to google. (csilvers) - -Fri Jan 14 16:53:32 2005 Google Inc. <opensource@google.com> - - * sparsehash: initial release: - The sparsehash package contains several hash-map implementations, - similar in API to SGI's hash_map class, but with different - performance characteristics. sparse_hash_map uses very little - space overhead: 1-2 bits per entry. dense_hash_map is typically - faster than the default SGI STL implementation. This package - also includes hash-set analogues of these classes. - +Thu Feb 23 23:47:18 2012 Google Inc. <google-sparsehash@googlegroups.com> + + * sparsehash: version 2.0.2 + * BUGFIX: Fix backwards compatibility for <google> include folders + +Wed Feb 01 02:57:48 2012 Google Inc. <google-sparsehash@googlegroups.com> + + * sparsehash: version 2.0.1 + * BUGFIX: Fix path to malloc_extension.h in time_hash_map.cc + +Tue Jan 31 11:33:04 2012 Google Inc. <google-sparsehash@googlegroups.com> + + * sparsehash: version 2.0 + * Renamed include directory from google/ to sparsehash/ (csilvers) + * Changed the 'official' sparsehash email in setup.py/etc + * Renamed google-sparsehash.sln to sparsehash.sln + * Changed copyright text to reflect Google's relinquished ownership + +Tue Dec 20 21:04:04 2011 Google Inc. <opensource@google.com> + + * sparsehash: version 1.12 release + * Add support for serializing/unserializing dense_hash_map/set to disk + * New simpler and more flexible serialization API + * Be more consistent about clearing on unserialize() even if it fails + * Quiet some compiler warnings about unused variables + * Add a timing test for iterating (suggested by google code issue 77) + * Add offset_to_pos, the opposite of pos_to_offset, to sparsetable + * PORTING: Add some missing #includes, needed on some systems + * Die at configure-time when g++ isn't installed + * Successfully make rpm's even when dpkg is missing + * Improve deleted key test in util/gtl/{dense,sparse}hashtable + * Update automake to 1.10.1, and autoconf to 2.62 + +Thu Jun 23 21:12:58 2011 Google Inc. <opensource@google.com> + + * sparsehash: version 1.11 release + * Improve performance on pointer keys by ignoring always-0 low bits + * Fix missing $(top_srcdir) in Makefile.am, which broke some compiles + * BUGFIX: Fix a crashing typo-bug in swap() + * PORTING: Remove support for old compilers that do not use 'std' + * Add some new benchmarks to test for a place dense_hash_* does badly + * Some cosmetic changes due to a switch to a new releasing tool + +Thu Jan 20 16:07:39 2011 Google Inc. <opensource@google.com> + + * sparsehash: version 1.10 release + * Follow ExtractKey return type, allowing it to return a reference + * PORTING: fix MSVC 10 warnings (constifying result_type, placement-new) + * Update from autoconf 2.61 to autoconf 2.65 + +Fri Sep 24 11:37:50 2010 Google Inc. <opensource@google.com> + + * sparsehash: version 1.9 release + * Add is_enum; make all enums PODs by default (romanp) + * Make find_or_insert() usable directly (dawidk) + * Use zero-memory trick for allocators to reduce space use (guilin) + * Fix some compiler warnings (chandlerc, eraman) + * BUGFIX: int -> size_type in one function we missed (csilvers) + * Added sparsehash.pc, for pkg-config (csilvers) + +Thu Jul 29 15:01:29 2010 Google Inc. <opensource@google.com> + + * sparsehash: version 1.8.1 release + * Remove -Werror from Makefile: gcc 4.3 gives spurious warnings + +Thu Jul 29 09:53:26 2010 Google Inc. <opensource@google.com> + + * sparsehash: version 1.8 release + * More support for Allocator, including allocator ctor arg (csilvers) + * Repack hasthable vars to reduce container size *more* (giao) + * Speed up clear() (csilvers) + * Change HT_{OCCUPANCY,SHRINK}_FLT from float to int (csilvers) + * Revamp test suite for more complete code & timing coverage (csilvers) + * BUGFIX: Enforce max_size for dense/sparse_hashtable (giao, csilvers) + * BUGFIX: Raise exception instead of crashing on overflow (csilvers) + * BUGFIX: Allow extraneous const in key type (csilvers) + * BUGFIX: Allow same functor for both hasher and key_equals (giao) + * PORTING: remove is_convertible, which gives AIX cc fits (csilvers) + * PORTING: Renamed README.windows to README_windows.txt (csilvers) + * Created non-empty NEWS file (csilvers) + +Wed Mar 31 12:32:03 2010 Google Inc. <opensource@google.com> + + * sparsehash: version 1.7 release + * Add support for Allocator (guilin) + * Add libc_allocator_with_realloc as the new default allocator (guilin) + * Repack {sparse,dense}hashtable vars to reduce container size (giao) + * BUGFIX: operator== no longer requires same table ordering (csilvers) + * BUGFIX: fix dense_hash_*(it,it) by requiring empty-key too (csilvers) + * PORTING: fix language bugs that gcc allowed (csilvers, chandlerc) + * Update from autoconf 2.61 to autoconf 2.64 + +Fri Jan 8 14:47:55 2010 Google Inc. <opensource@google.com> + + * sparsehash: version 1.6 release + * New accessor methods for deleted_key, empty_key (sjackman) + * Use explicit hash functions in sparsehash tests (csilvers) + * BUGFIX: Cast resize to fix SUNWspro bug (csilvers) + * Check for sz overflow in min_size (csilvers) + * Speed up clear() for dense and sparse hashtables (jeff) + * Avoid shrinking in all cases when min-load is 0 (shaunj, csilvers) + * Improve densehashtable code for the deleted key (gpike) + * BUGFIX: Fix operator= when the 2 empty-keys differ (andreidam) + * BUGFIX: Fix ht copying when empty-key isn't set (andreidam) + * PORTING: Use TmpFile() instead of /tmp on MinGW (csilvers) + * PORTING: Use filenames that work with Stratus VOS. + +Tue May 12 14:16:38 2009 Google Inc. <opensource@google.com> + + * sparsehash: version 1.5.2 release + * Fix compile error: not initializing set_key in all constructors + +Fri May 8 15:23:44 2009 Google Inc. <opensource@google.com> + + * sparsehash: version 1.5.1 release + * Fix broken equal_range() for all the hash-classes (csilvers) + +Wed May 6 11:28:49 2009 Google Inc. <opensource@google.com> + + * sparsehash: version 1.5 release + * Support the tr1 unordered_map (and unordered_set) API (csilvers) + * Store only key for delkey; reduces need for 0-arg c-tor (csilvers) + * Prefer unordered_map to hash_map for the timing test (csilvers) + * PORTING: update the resource use for 64-bit machines (csilvers) + * PORTING: fix MIN/MAX collisions by un-#including windows.h (csilvers) + * Updated autoconf version to 2.61 and libtool version to 1.5.26 + +Wed Jan 28 17:11:31 2009 Google Inc. <opensource@google.com> + + * sparsehash: version 1.4 release + * Allow hashtables to be <32 buckets (csilvers) + * Fix initial-sizing bug: was sizing tables too small (csilvers) + * Add asserts that clients don't abuse deleted/empty key (csilvers) + * Improve determination of 32/64 bit for C code (csilvers) + * Small fix for doc files in rpm (csilvers) + +Thu Nov 6 15:06:09 2008 Google Inc. <opensource@google.com> + + * sparsehash: version 1.3 release + * Add an interface to change the parameters for resizing (myl) + * Document another potentially good hash function (csilvers) + +Thu Sep 18 13:53:20 2008 Google Inc. <opensource@google.com> + + * sparsehash: version 1.2 release + * Augment documentation to better describe namespace issues (csilvers) + * BUG FIX: replace hash<> with SPARSEHASH_HASH, for windows (csilvers) + * Add timing test to unittest to test repeated add+delete (csilvers) + * Do better picking a new size when resizing (csilvers) + * Use ::google instead of google as a namespace (csilvers) + * Improve threading test at config time (csilvers) + +Mon Feb 11 16:30:11 2008 Google Inc. <opensource@google.com> + + * sparsehash: version 1.1 release + * Fix brown-paper-bag bug in some constructors (rafferty) + * Fix problem with variables shadowing member vars, add -Wshadow + +Thu Nov 29 11:44:38 2007 Google Inc. <opensource@google.com> + + * sparsehash: version 1.0.2 release + * Fix a final reference to hash<> to use SPARSEHASH_HASH<> instead. + +Wed Nov 14 08:47:48 2007 Google Inc. <opensource@google.com> + + * sparsehash: version 1.0.1 release :-( + * Remove an unnecessary (harmful) "#define hash" in windows' config.h + +Tue Nov 13 15:15:46 2007 Google Inc. <opensource@google.com> + + * sparsehash: version 1.0 release! We are now out of beta. + * Clean up Makefile awk script to be more readable (csilvers) + * Namespace fixes: use fewer #defines, move typedefs into namespace + +Fri Oct 12 12:35:24 2007 Google Inc. <opensource@google.com> + + * sparsehash: version 0.9.1 release + * Fix Makefile awk script to work on more architectures (csilvers) + * Add test to test code in more 'real life' situations (csilvers) + +Tue Oct 9 14:15:21 2007 Google Inc. <opensource@google.com> + + * sparsehash: version 0.9 release + * More type-hygiene improvements, especially for 64-bit (csilvers) + * Some configure improvements to improve portability, utility (austern) + * Small bugfix for operator== for dense_hash_map (jeff) + +Tue Jul 3 12:55:04 2007 Google Inc. <opensource@google.com> + + * sparsehash: version 0.8 release + * Minor type-hygiene improvements: size_t for int, etc. (csilvers) + * Porting improvements: tests pass on OS X, FreeBSD, Solaris (csilvers) + * Full windows port! VS solution provided for all unittests (csilvers) + +Mon Jun 11 11:33:41 2007 Google Inc. <opensource@google.com> + + * sparsehash: version 0.7 release + * Syntax fixes to better support gcc 4.3 and VC++ 7 (mec, csilvers) + * Improved windows/VC++ support (see README.windows) (csilvers) + * Config improvements: better tcmalloc support and config.h (csilvers) + * More robust with missing hash_map + nix 'trampoline' .h's (csilvers) + * Support for STLport's hash_map/hash_fun locations (csilvers) + * Add .m4 files to distribution; now all source is there (csilvers) + * Tiny modification of shrink-threshhold to allow never-shrinking (amc) + * Protect timing tests against aggressive optimizers (csilvers) + * Extend time_hash_map to test bigger objects (csilvers) + * Extend type-trait support to work with const objects (csilvers) + * USER VISIBLE: speed up all code by replacing memmove with memcpy + (csilvers) + +Tue Mar 20 17:29:34 2007 Google Inc. <opensource@google.com> + + * sparsehash: version 0.6 release + * Some improvement to type-traits (jyasskin) + * Better timing results when google-perftools is installed (sanjay) + * Updates and fixes to html documentation and README (csilvers) + * A bit more careful about #includes (csilvers) + * Fix for typo that broken compilation on some systems (csilvers) + * USER VISIBLE: New clear_no_resize() method added to dense_hash_map + (uszkoreit) + +Sat Oct 21 13:47:47 2006 Google Inc. <opensource@google.com> + + * sparsehash: version 0.5 release + * Support uint16_t (SunOS) in addition to u_int16_t (BSD) (csilvers) + * Get rid of UNDERSTANDS_ITERATOR_TAGS; everyone understands (csilvers) + * Test that empty-key and deleted-key differ (rbayardo) + * Fix example docs: strcmp needs to test for NULL (csilvers) + +Sun Apr 23 22:42:35 2006 Google Inc. <opensource@google.com> + + * sparsehash: version 0.4 release + * Remove POD requirement for keys and values! (austern) + * Add tr1-compatible type-traits system to speed up POD ops. (austern) + * Fixed const-iterator bug where postfix ++ didn't compile. (csilvers) + * Fixed iterator comparison bugs where <= was incorrect. (csilvers) + * Clean up config.h to keep its #defines from conflicting. (csilvers) + * Big documentation sweep and cleanup. (csilvers) + * Update documentation to talk more about good hash fns. (csilvers) + * Fixes to compile on MSVC (working around some MSVC bugs). (rennie) + * Avoid resizing hashtable on operator[] lookups (austern) + +Thu Nov 3 20:12:31 2005 Google Inc. <opensource@google.com> + + * sparsehash: version 0.3 release + * Quiet compiler warnings on some compilers. (csilvers) + * Some documentation fixes: example code for dense_hash_map. (csilvers) + * Fix a bug where swap() wasn't swapping delete_key(). (csilvers) + * set_deleted_key() and set_empty_key() now take a key only, + allowing hash-map values to be forward-declared. (csilvers) + * support for std::insert_iterator (and std::inserter). (csilvers) + +Mon May 2 07:04:46 2005 Google Inc. <opensource@google.com> + + * sparsehash: version 0.2 release + * Preliminary support for msvc++ compilation. (csilvers) + * Documentation fixes -- some example code was incomplete! (csilvers) + * Minimize size of config.h to avoid other-package conflicts (csilvers) + * Contribute a C-based version of sparsehash that served as the + inspiration for this code. One day, I hope to clean it up and + support it, but for now it's just in experimental/, for playing + around with. (csilvers) + * Change default namespace from std to google. (csilvers) + +Fri Jan 14 16:53:32 2005 Google Inc. <opensource@google.com> + + * sparsehash: initial release: + The sparsehash package contains several hash-map implementations, + similar in API to SGI's hash_map class, but with different + performance characteristics. sparse_hash_map uses very little + space overhead: 1-2 bits per entry. dense_hash_map is typically + faster than the default SGI STL implementation. This package + also includes hash-set analogues of these classes. + diff --git a/contrib/libs/sparsehash/NEWS b/contrib/libs/sparsehash/NEWS index c97f503bff..4af929c394 100644 --- a/contrib/libs/sparsehash/NEWS +++ b/contrib/libs/sparsehash/NEWS @@ -1,193 +1,193 @@ == 12 October 2015 == - + Various small fixes to ensure compilation on modern compilers and operating systems. Tagged as 2.0.3 == 23 February 2012 == -A backwards incompatibility arose from flattening the include headers -structure for the <google> folder. - -This is now fixed in 2.0.2. You only need to upgrade if you had previously -included files from the <google/sparsehash> folder. - -== 1 February 2012 == - -A minor bug related to the namespace switch from google to sparsehash -stopped the build from working when perftools is also installed. - -This is now fixed in 2.0.1. You only need to upgrade if you have perftools -installed. - -== 31 January 2012 == - -I've just released sparsehash 2.0. - -The `google-sparsehash` project has been renamed to `sparsehash`. I -(csilvers) am stepping down as maintainer, to be replaced by the team -of Donovan Hide and Geoff Pike. Welcome to the team, Donovan and -Geoff! Donovan has been an active contributor to sparsehash bug -reports and discussions in the past, and Geoff has been closely -involved with sparsehash inside Google (in addition to writing the -[http://code.google.com/p/cityhash CityHash hash function]). The two -of them together should be a formidable force. For good. - -I bumped the major version number up to 2 to reflect the new community -ownership of the project. All the -[http://sparsehash.googlecode.com/svn/tags/sparsehash-2.0/ChangeLog changes] -are related to the renaming. - -The only functional change from sparsehash 1.12 is that I've renamed -the `google/` include-directory to be `sparsehash/` instead. New code -should `#include <sparsehash/sparse_hash_map>`/etc. I've kept the old -names around as forwarding headers to the new, so `#include -<google/sparse_hash_map>` will continue to work. - -Note that the classes and functions remain in the `google` C++ -namespace (I didn't change that to `sparsehash` as well); I think -that's a trickier transition, and can happen in a future release. - - -=== 18 January 2011 === - -The `google-sparsehash` Google Code page has been renamed to -`sparsehash`, in preparation for the project being renamed to -`sparsehash`. In the coming weeks, I'll be stepping down as -maintainer for the sparsehash project, and as part of that Google is -relinquishing ownership of the project; it will now be entirely -community run. The name change reflects that shift. - - -=== 20 December 2011 === - -I've just released sparsehash 1.12. This release features improved -I/O (serialization) support. Support is finally added to serialize -and unserialize `dense_hash_map`/`set`, paralleling the existing code -for `sparse_hash_map`/`set`. In addition, the serialization API has -gotten simpler, with a single `serialize()` method to write to disk, -and an `unserialize()` method to read from disk. Finally, support has -gotten more generic, with built-in support for both C `FILE*`s and C++ -streams, and an extension mechanism to support arbitrary sources and -sinks. - -There are also more minor changes, including minor bugfixes, an -improved deleted-key test, and a minor addition to the `sparsetable` -API. See the [http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.12/ChangeLog ChangeLog] -for full details. - -=== 23 June 2011 === - -I've just released sparsehash 1.11. The major user-visible change is -that the default behavior is improved -- using the hash_map/set is -faster -- for hashtables where the key is a pointer. We now notice -that case and ignore the low 2-3 bits (which are almost always 0 for -pointers) when hashing. - -Another user-visible change is we've removed the tests for whether the -STL (vector, pair, etc) is defined in the 'std' namespace. gcc 2.95 -is the most recent compiler I know of to put STL types and functions -in the global namespace. If you need to use such an old compiler, do -not update to the latest sparsehash release. - -We've also changed the internal tools we use to integrate -Googler-supplied patches to sparsehash into the opensource release. -These new tools should result in more frequent updates with better -change descriptions. They will also result in future ChangeLog -entries being much more verbose (for better or for worse). - -A full list of changes is described in -[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.11/ChangeLog ChangeLog]. - -=== 21 January 2011 === - -I've just released sparsehash 1.10. This fixes a performance -regression in sparsehash 1.8, where sparse_hash_map would copy -hashtable keys by value even when the key was explicitly a reference. -It also fixes compiler warnings from MSVC 10, which uses some c++0x -features that did not interact well with sparsehash. - -There is no reason to upgrade unless you use references for your -hashtable keys, or compile with MSVC 10. A full list of changes is -described in -[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.10/ChangeLog ChangeLog]. - - -=== 24 September 2010 === - -I've just released sparsehash 1.9. This fixes a size regression in -sparsehash 1.8, where the new allocator would take up space in -`sparse_hash_map`, doubling the sparse_hash_map overhead (from 1-2 -bits per bucket to 3 or so). All users are encouraged to upgrade. - -This change also marks enums as being Plain Old Data, which can speed -up hashtables with enum keys and/or values. A full list of changes is -described in -[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.9/ChangeLog ChangeLog]. - -=== 29 July 2010 === - -I've just released sparsehash 1.8. This includes improved support for -`Allocator`, including supporting the allocator constructor arg and -`get_allocator()` access method. - -To work around a bug in gcc 4.0.x, I've renamed the static variables -`HT_OCCUPANCY_FLT` and `HT_SHRINK_FLT` to `HT_OCCUPANCY_PCT` and -`HT_SHRINK_PCT`, and changed their type from float to int. This -should not be a user-visible change, since these variables are only -used in the internal hashtable classes (sparsehash clients should use -`max_load_factor()` and `min_load_factor()` instead of modifying these -static variables), but if you do access these constants, you will need -to change your code. - -Internally, the biggest change is a revamp of the test suite. It now -has more complete coverage, and a more capable timing tester. There -are other, more minor changes as well. A full list of changes is -described in the -[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.8/ChangeLog ChangeLog]. - -=== 31 March 2010 === - -I've just released sparsehash 1.7. The major news here is the -addition of `Allocator` support. Previously, these hashtable classes -would just ignore the `Allocator` template parameter. They now -respect it, and even inherit `size_type`, `pointer`, etc. from the -allocator class. By default, they use a special allocator we provide -that uses libc `malloc` and `free` to allocate. The hash classes -notice when this special allocator is being used, and use `realloc` -when it can. This means that the default allocator is significantly -faster than custom allocators are likely to be (since realloc-like -functionality is not supported by STL allocators). - -There are a few more minor changes as well. A full list of changes is -described in the -[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.7/ChangeLog ChangeLog]. - -=== 11 January 2010 === - -I've just released sparsehash 1.6. The API has widened a bit with the -addition of `deleted_key()` and `empty_key()`, which let you query -what values these keys have. A few rather obscure bugs have been -fixed (such as an error when copying one hashtable into another when -the empty_keys differ). A full list of changes is described in the -[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.6/ChangeLog ChangeLog]. - -=== 9 May 2009 === - -I've just released sparsehash 1.5.1. Hot on the heels of sparsehash -1.5, this release fixes a longstanding bug in the sparsehash code, -where `equal_range` would always return an empty range. It now works -as documented. All sparsehash users are encouraged to upgrade. - -=== 7 May 2009 === - -I've just released sparsehash 1.5. This release introduces tr1 -compatibility: I've added `rehash`, `begin(i)`, and other methods that -are expected to be part of the `unordered_map` API once `tr1` in -introduced. This allows `sparse_hash_map`, `dense_hash_map`, -`sparse_hash_set`, and `dense_hash_set` to be (almost) drop-in -replacements for `unordered_map` and `unordered_set`. - -There is no need to upgrade unless you need this functionality, or -need one of the other, more minor, changes described in the -[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.5/ChangeLog ChangeLog]. - +A backwards incompatibility arose from flattening the include headers +structure for the <google> folder. + +This is now fixed in 2.0.2. You only need to upgrade if you had previously +included files from the <google/sparsehash> folder. + +== 1 February 2012 == + +A minor bug related to the namespace switch from google to sparsehash +stopped the build from working when perftools is also installed. + +This is now fixed in 2.0.1. You only need to upgrade if you have perftools +installed. + +== 31 January 2012 == + +I've just released sparsehash 2.0. + +The `google-sparsehash` project has been renamed to `sparsehash`. I +(csilvers) am stepping down as maintainer, to be replaced by the team +of Donovan Hide and Geoff Pike. Welcome to the team, Donovan and +Geoff! Donovan has been an active contributor to sparsehash bug +reports and discussions in the past, and Geoff has been closely +involved with sparsehash inside Google (in addition to writing the +[http://code.google.com/p/cityhash CityHash hash function]). The two +of them together should be a formidable force. For good. + +I bumped the major version number up to 2 to reflect the new community +ownership of the project. All the +[http://sparsehash.googlecode.com/svn/tags/sparsehash-2.0/ChangeLog changes] +are related to the renaming. + +The only functional change from sparsehash 1.12 is that I've renamed +the `google/` include-directory to be `sparsehash/` instead. New code +should `#include <sparsehash/sparse_hash_map>`/etc. I've kept the old +names around as forwarding headers to the new, so `#include +<google/sparse_hash_map>` will continue to work. + +Note that the classes and functions remain in the `google` C++ +namespace (I didn't change that to `sparsehash` as well); I think +that's a trickier transition, and can happen in a future release. + + +=== 18 January 2011 === + +The `google-sparsehash` Google Code page has been renamed to +`sparsehash`, in preparation for the project being renamed to +`sparsehash`. In the coming weeks, I'll be stepping down as +maintainer for the sparsehash project, and as part of that Google is +relinquishing ownership of the project; it will now be entirely +community run. The name change reflects that shift. + + +=== 20 December 2011 === + +I've just released sparsehash 1.12. This release features improved +I/O (serialization) support. Support is finally added to serialize +and unserialize `dense_hash_map`/`set`, paralleling the existing code +for `sparse_hash_map`/`set`. In addition, the serialization API has +gotten simpler, with a single `serialize()` method to write to disk, +and an `unserialize()` method to read from disk. Finally, support has +gotten more generic, with built-in support for both C `FILE*`s and C++ +streams, and an extension mechanism to support arbitrary sources and +sinks. + +There are also more minor changes, including minor bugfixes, an +improved deleted-key test, and a minor addition to the `sparsetable` +API. See the [http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.12/ChangeLog ChangeLog] +for full details. + +=== 23 June 2011 === + +I've just released sparsehash 1.11. The major user-visible change is +that the default behavior is improved -- using the hash_map/set is +faster -- for hashtables where the key is a pointer. We now notice +that case and ignore the low 2-3 bits (which are almost always 0 for +pointers) when hashing. + +Another user-visible change is we've removed the tests for whether the +STL (vector, pair, etc) is defined in the 'std' namespace. gcc 2.95 +is the most recent compiler I know of to put STL types and functions +in the global namespace. If you need to use such an old compiler, do +not update to the latest sparsehash release. + +We've also changed the internal tools we use to integrate +Googler-supplied patches to sparsehash into the opensource release. +These new tools should result in more frequent updates with better +change descriptions. They will also result in future ChangeLog +entries being much more verbose (for better or for worse). + +A full list of changes is described in +[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.11/ChangeLog ChangeLog]. + +=== 21 January 2011 === + +I've just released sparsehash 1.10. This fixes a performance +regression in sparsehash 1.8, where sparse_hash_map would copy +hashtable keys by value even when the key was explicitly a reference. +It also fixes compiler warnings from MSVC 10, which uses some c++0x +features that did not interact well with sparsehash. + +There is no reason to upgrade unless you use references for your +hashtable keys, or compile with MSVC 10. A full list of changes is +described in +[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.10/ChangeLog ChangeLog]. + + +=== 24 September 2010 === + +I've just released sparsehash 1.9. This fixes a size regression in +sparsehash 1.8, where the new allocator would take up space in +`sparse_hash_map`, doubling the sparse_hash_map overhead (from 1-2 +bits per bucket to 3 or so). All users are encouraged to upgrade. + +This change also marks enums as being Plain Old Data, which can speed +up hashtables with enum keys and/or values. A full list of changes is +described in +[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.9/ChangeLog ChangeLog]. + +=== 29 July 2010 === + +I've just released sparsehash 1.8. This includes improved support for +`Allocator`, including supporting the allocator constructor arg and +`get_allocator()` access method. + +To work around a bug in gcc 4.0.x, I've renamed the static variables +`HT_OCCUPANCY_FLT` and `HT_SHRINK_FLT` to `HT_OCCUPANCY_PCT` and +`HT_SHRINK_PCT`, and changed their type from float to int. This +should not be a user-visible change, since these variables are only +used in the internal hashtable classes (sparsehash clients should use +`max_load_factor()` and `min_load_factor()` instead of modifying these +static variables), but if you do access these constants, you will need +to change your code. + +Internally, the biggest change is a revamp of the test suite. It now +has more complete coverage, and a more capable timing tester. There +are other, more minor changes as well. A full list of changes is +described in the +[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.8/ChangeLog ChangeLog]. + +=== 31 March 2010 === + +I've just released sparsehash 1.7. The major news here is the +addition of `Allocator` support. Previously, these hashtable classes +would just ignore the `Allocator` template parameter. They now +respect it, and even inherit `size_type`, `pointer`, etc. from the +allocator class. By default, they use a special allocator we provide +that uses libc `malloc` and `free` to allocate. The hash classes +notice when this special allocator is being used, and use `realloc` +when it can. This means that the default allocator is significantly +faster than custom allocators are likely to be (since realloc-like +functionality is not supported by STL allocators). + +There are a few more minor changes as well. A full list of changes is +described in the +[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.7/ChangeLog ChangeLog]. + +=== 11 January 2010 === + +I've just released sparsehash 1.6. The API has widened a bit with the +addition of `deleted_key()` and `empty_key()`, which let you query +what values these keys have. A few rather obscure bugs have been +fixed (such as an error when copying one hashtable into another when +the empty_keys differ). A full list of changes is described in the +[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.6/ChangeLog ChangeLog]. + +=== 9 May 2009 === + +I've just released sparsehash 1.5.1. Hot on the heels of sparsehash +1.5, this release fixes a longstanding bug in the sparsehash code, +where `equal_range` would always return an empty range. It now works +as documented. All sparsehash users are encouraged to upgrade. + +=== 7 May 2009 === + +I've just released sparsehash 1.5. This release introduces tr1 +compatibility: I've added `rehash`, `begin(i)`, and other methods that +are expected to be part of the `unordered_map` API once `tr1` in +introduced. This allows `sparse_hash_map`, `dense_hash_map`, +`sparse_hash_set`, and `dense_hash_set` to be (almost) drop-in +replacements for `unordered_map` and `unordered_set`. + +There is no need to upgrade unless you need this functionality, or +need one of the other, more minor, changes described in the +[http://google-sparsehash.googlecode.com/svn/tags/sparsehash-1.5/ChangeLog ChangeLog]. + diff --git a/contrib/libs/sparsehash/README b/contrib/libs/sparsehash/README index 7c7dc26f77..26bb485008 100644 --- a/contrib/libs/sparsehash/README +++ b/contrib/libs/sparsehash/README @@ -1,147 +1,147 @@ -This directory contains several hash-map implementations, similar in -API to SGI's hash_map class, but with different performance -characteristics. sparse_hash_map uses very little space overhead, 1-2 -bits per entry. dense_hash_map is very fast, particulary on lookup. -(sparse_hash_set and dense_hash_set are the set versions of these -routines.) On the other hand, these classes have requirements that -may not make them appropriate for all applications. - -All these implementation use a hashtable with internal quadratic -probing. This method is space-efficient -- there is no pointer -overhead -- and time-efficient for good hash functions. - -COMPILING ---------- -To compile test applications with these classes, run ./configure -followed by make. To install these header files on your system, run -'make install'. (On Windows, the instructions are different; see -README_windows.txt.) See INSTALL for more details. - -This code should work on any modern C++ system. It has been tested on -Linux (Ubuntu, Fedora, RedHat, Debian), Solaris 10 x86, FreeBSD 6.0, -OS X 10.3 and 10.4, and Windows under both VC++7 and VC++8. - -USING ------ -See the html files in the doc directory for small example programs -that use these classes. It's enough to just include the header file: - - #include <sparsehash/sparse_hash_map> // or sparse_hash_set, dense_hash_map, ... - google::sparse_hash_set<int, int> number_mapper; - -and use the class the way you would other hash-map implementations. -(Though see "API" below for caveats.) - -By default (you can change it via a flag to ./configure), these hash -implementations are defined in the google namespace. - -API ---- -The API for sparse_hash_map, dense_hash_map, sparse_hash_set, and -dense_hash_set, are a superset of the API of SGI's hash_map class. -See doc/sparse_hash_map.html, et al., for more information about the -API. - -The usage of these classes differ from SGI's hash_map, and other -hashtable implementations, in the following major ways: - -1) dense_hash_map requires you to set aside one key value as the - 'empty bucket' value, set via the set_empty_key() method. This - *MUST* be called before you can use the dense_hash_map. It is - illegal to insert any elements into a dense_hash_map whose key is - equal to the empty-key. - -2) For both dense_hash_map and sparse_hash_map, if you wish to delete - elements from the hashtable, you must set aside a key value as the - 'deleted bucket' value, set via the set_deleted_key() method. If - your hash-map is insert-only, there is no need to call this - method. If you call set_deleted_key(), it is illegal to insert any - elements into a dense_hash_map or sparse_hash_map whose key is - equal to the deleted-key. - -3) These hash-map implementation support I/O. See below. - -There are also some smaller differences: - -1) The constructor takes an optional argument that specifies the - number of elements you expect to insert into the hashtable. This - differs from SGI's hash_map implementation, which takes an optional - number of buckets. - -2) erase() does not immediately reclaim memory. As a consequence, - erase() does not invalidate any iterators, making loops like this - correct: - for (it = ht.begin(); it != ht.end(); ++it) - if (...) ht.erase(it); - As another consequence, a series of erase() calls can leave your - hashtable using more memory than it needs to. The hashtable will - automatically compact at the next call to insert(), but to - manually compact a hashtable, you can call - ht.resize(0) - -I/O ---- -In addition to the normal hash-map operations, sparse_hash_map can -read and write hashtables to disk. (dense_hash_map also has the API, -but it has not yet been implemented, and writes will always fail.) - -In the simplest case, writing a hashtable is as easy as calling two -methods on the hashtable: - ht.write_metadata(fp); - ht.write_nopointer_data(fp); - -Reading in this data is equally simple: - google::sparse_hash_map<...> ht; - ht.read_metadata(fp); - ht.read_nopointer_data(fp); - -The above is sufficient if the key and value do not contain any -pointers: they are basic C types or agglomorations of basic C types. -If the key and/or value do contain pointers, you can still store the -hashtable by replacing write_nopointer_data() with a custom writing -routine. See sparse_hash_map.html et al. for more information. - -SPARSETABLE ------------ -In addition to the hash-map and hash-set classes, this package also -provides sparsetable.h, an array implementation that uses space -proportional to the number of elements in the array, rather than the +This directory contains several hash-map implementations, similar in +API to SGI's hash_map class, but with different performance +characteristics. sparse_hash_map uses very little space overhead, 1-2 +bits per entry. dense_hash_map is very fast, particulary on lookup. +(sparse_hash_set and dense_hash_set are the set versions of these +routines.) On the other hand, these classes have requirements that +may not make them appropriate for all applications. + +All these implementation use a hashtable with internal quadratic +probing. This method is space-efficient -- there is no pointer +overhead -- and time-efficient for good hash functions. + +COMPILING +--------- +To compile test applications with these classes, run ./configure +followed by make. To install these header files on your system, run +'make install'. (On Windows, the instructions are different; see +README_windows.txt.) See INSTALL for more details. + +This code should work on any modern C++ system. It has been tested on +Linux (Ubuntu, Fedora, RedHat, Debian), Solaris 10 x86, FreeBSD 6.0, +OS X 10.3 and 10.4, and Windows under both VC++7 and VC++8. + +USING +----- +See the html files in the doc directory for small example programs +that use these classes. It's enough to just include the header file: + + #include <sparsehash/sparse_hash_map> // or sparse_hash_set, dense_hash_map, ... + google::sparse_hash_set<int, int> number_mapper; + +and use the class the way you would other hash-map implementations. +(Though see "API" below for caveats.) + +By default (you can change it via a flag to ./configure), these hash +implementations are defined in the google namespace. + +API +--- +The API for sparse_hash_map, dense_hash_map, sparse_hash_set, and +dense_hash_set, are a superset of the API of SGI's hash_map class. +See doc/sparse_hash_map.html, et al., for more information about the +API. + +The usage of these classes differ from SGI's hash_map, and other +hashtable implementations, in the following major ways: + +1) dense_hash_map requires you to set aside one key value as the + 'empty bucket' value, set via the set_empty_key() method. This + *MUST* be called before you can use the dense_hash_map. It is + illegal to insert any elements into a dense_hash_map whose key is + equal to the empty-key. + +2) For both dense_hash_map and sparse_hash_map, if you wish to delete + elements from the hashtable, you must set aside a key value as the + 'deleted bucket' value, set via the set_deleted_key() method. If + your hash-map is insert-only, there is no need to call this + method. If you call set_deleted_key(), it is illegal to insert any + elements into a dense_hash_map or sparse_hash_map whose key is + equal to the deleted-key. + +3) These hash-map implementation support I/O. See below. + +There are also some smaller differences: + +1) The constructor takes an optional argument that specifies the + number of elements you expect to insert into the hashtable. This + differs from SGI's hash_map implementation, which takes an optional + number of buckets. + +2) erase() does not immediately reclaim memory. As a consequence, + erase() does not invalidate any iterators, making loops like this + correct: + for (it = ht.begin(); it != ht.end(); ++it) + if (...) ht.erase(it); + As another consequence, a series of erase() calls can leave your + hashtable using more memory than it needs to. The hashtable will + automatically compact at the next call to insert(), but to + manually compact a hashtable, you can call + ht.resize(0) + +I/O +--- +In addition to the normal hash-map operations, sparse_hash_map can +read and write hashtables to disk. (dense_hash_map also has the API, +but it has not yet been implemented, and writes will always fail.) + +In the simplest case, writing a hashtable is as easy as calling two +methods on the hashtable: + ht.write_metadata(fp); + ht.write_nopointer_data(fp); + +Reading in this data is equally simple: + google::sparse_hash_map<...> ht; + ht.read_metadata(fp); + ht.read_nopointer_data(fp); + +The above is sufficient if the key and value do not contain any +pointers: they are basic C types or agglomorations of basic C types. +If the key and/or value do contain pointers, you can still store the +hashtable by replacing write_nopointer_data() with a custom writing +routine. See sparse_hash_map.html et al. for more information. + +SPARSETABLE +----------- +In addition to the hash-map and hash-set classes, this package also +provides sparsetable.h, an array implementation that uses space +proportional to the number of elements in the array, rather than the maximum element index. It uses very little space overhead: 2 to 5 bits per entry. See doc/sparsetable.html for the API. - -RESOURCE USAGE --------------- + +RESOURCE USAGE +-------------- * sparse_hash_map has memory overhead of about 4 to 10 bits per hash-map entry, assuming a typical average occupancy of 50%. -* dense_hash_map has a factor of 2-3 memory overhead: if your - hashtable data takes X bytes, dense_hash_map will use 3X-4X memory - total. - -Hashtables tend to double in size when resizing, creating an -additional 50% space overhead. dense_hash_map does in fact have a +* dense_hash_map has a factor of 2-3 memory overhead: if your + hashtable data takes X bytes, dense_hash_map will use 3X-4X memory + total. + +Hashtables tend to double in size when resizing, creating an +additional 50% space overhead. dense_hash_map does in fact have a significant "high water mark" memory use requirement, which is 6 times the size of hash entries in the table when resizing (when reaching 50% occupancy, the table resizes to double the previous size, and the old table (2x) is copied to the new table (4x)). -sparse_hash_map, however, is written to need very little space -overhead when resizing: only a few bits per hashtable entry. - -PERFORMANCE ------------ -You can compile and run the included file time_hash_map.cc to examine -the performance of sparse_hash_map, dense_hash_map, and your native -hash_map implementation on your system. One test against the -SGI hash_map implementation gave the following timing information for -a simple find() call: - SGI hash_map: 22 ns - dense_hash_map: 13 ns - sparse_hash_map: 117 ns - SGI map: 113 ns - -See doc/performance.html for more detailed charts on resource usage -and performance data. - ---- -16 March 2005 -(Last updated: 12 September 2010) +sparse_hash_map, however, is written to need very little space +overhead when resizing: only a few bits per hashtable entry. + +PERFORMANCE +----------- +You can compile and run the included file time_hash_map.cc to examine +the performance of sparse_hash_map, dense_hash_map, and your native +hash_map implementation on your system. One test against the +SGI hash_map implementation gave the following timing information for +a simple find() call: + SGI hash_map: 22 ns + dense_hash_map: 13 ns + sparse_hash_map: 117 ns + SGI map: 113 ns + +See doc/performance.html for more detailed charts on resource usage +and performance data. + +--- +16 March 2005 +(Last updated: 12 September 2010) diff --git a/contrib/libs/sparsehash/README_windows.txt b/contrib/libs/sparsehash/README_windows.txt index 99234e02fa..54df6f8eac 100644 --- a/contrib/libs/sparsehash/README_windows.txt +++ b/contrib/libs/sparsehash/README_windows.txt @@ -1,25 +1,25 @@ -This project has been ported to Windows. A working solution file -exists in this directory: - sparsehash.sln - -You can load this solution file into either VC++ 7.1 (Visual Studio -2003) or VC++ 8.0 (Visual Studio 2005) -- in the latter case, it will -automatically convert the files to the latest format for you. - -When you build the solution, it will create a number of -unittests,which you can run by hand (or, more easily, under the Visual -Studio debugger) to make sure everything is working properly on your -system. The binaries will end up in a directory called "debug" or -"release" in the top-level directory (next to the .sln file). - -Note that these systems are set to build in Debug mode by default. -You may want to change them to Release mode. - -I have little experience with Windows programming, so there may be -better ways to set this up than I've done! If you run across any -problems, please post to the google-sparsehash Google Group, or report -them on the sparsehash Google Code site: - http://groups.google.com/group/google-sparsehash - http://code.google.com/p/sparsehash/issues/list - --- craig +This project has been ported to Windows. A working solution file +exists in this directory: + sparsehash.sln + +You can load this solution file into either VC++ 7.1 (Visual Studio +2003) or VC++ 8.0 (Visual Studio 2005) -- in the latter case, it will +automatically convert the files to the latest format for you. + +When you build the solution, it will create a number of +unittests,which you can run by hand (or, more easily, under the Visual +Studio debugger) to make sure everything is working properly on your +system. The binaries will end up in a directory called "debug" or +"release" in the top-level directory (next to the .sln file). + +Note that these systems are set to build in Debug mode by default. +You may want to change them to Release mode. + +I have little experience with Windows programming, so there may be +better ways to set this up than I've done! If you run across any +problems, please post to the google-sparsehash Google Group, or report +them on the sparsehash Google Code site: + http://groups.google.com/group/google-sparsehash + http://code.google.com/p/sparsehash/issues/list + +-- craig diff --git a/contrib/libs/sparsehash/TODO b/contrib/libs/sparsehash/TODO index 50476bd06f..e9b0263cf5 100644 --- a/contrib/libs/sparsehash/TODO +++ b/contrib/libs/sparsehash/TODO @@ -1,28 +1,28 @@ -1) TODO: I/O implementation in densehashtable.h - -2) TODO: document SPARSEHASH_STAT_UPDATE macro, and also macros that - tweak performance. Perhaps add support to these to the API? - -3) TODO: support exceptions? - -4) BUG: sparsetable's operator[] doesn't work well with printf: you - need to explicitly cast the result to value_type to print it. (It - works fine with streams.) - -5) TODO: consider rewriting dense_hash_map to use a 'groups' scheme, - like sparsetable, but without the sparse-allocation within a - group. This makes resizing have better memory-use properties. The - downside is that probes across groups might take longer since - groups are not contiguous in memory. Making groups the same size - as a cache-line, and ensuring they're loaded on cache-line - boundaries, might help. Needs careful testing to make sure it - doesn't hurt performance. - -6) TODO: Get the C-only version of sparsehash in experimental/ ready - for prime-time. - -7) TODO: use cmake (www.cmake.org) to make it easy to isntall this on - a windows system. - ---- -28 February 2007 +1) TODO: I/O implementation in densehashtable.h + +2) TODO: document SPARSEHASH_STAT_UPDATE macro, and also macros that + tweak performance. Perhaps add support to these to the API? + +3) TODO: support exceptions? + +4) BUG: sparsetable's operator[] doesn't work well with printf: you + need to explicitly cast the result to value_type to print it. (It + works fine with streams.) + +5) TODO: consider rewriting dense_hash_map to use a 'groups' scheme, + like sparsetable, but without the sparse-allocation within a + group. This makes resizing have better memory-use properties. The + downside is that probes across groups might take longer since + groups are not contiguous in memory. Making groups the same size + as a cache-line, and ensuring they're loaded on cache-line + boundaries, might help. Needs careful testing to make sure it + doesn't hurt performance. + +6) TODO: Get the C-only version of sparsehash in experimental/ ready + for prime-time. + +7) TODO: use cmake (www.cmake.org) to make it easy to isntall this on + a windows system. + +--- +28 February 2007 diff --git a/contrib/libs/sparsehash/src/sparsehash/dense_hash_map b/contrib/libs/sparsehash/src/sparsehash/dense_hash_map index 693ada163f..05fd580e64 100644 --- a/contrib/libs/sparsehash/src/sparsehash/dense_hash_map +++ b/contrib/libs/sparsehash/src/sparsehash/dense_hash_map @@ -1,369 +1,369 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// ---- -// -// This is just a very thin wrapper over densehashtable.h, just -// like sgi stl's stl_hash_map is a very thin wrapper over -// stl_hashtable. The major thing we define is operator[], because -// we have a concept of a data_type which stl_hashtable doesn't -// (it only has a key and a value). -// -// NOTE: this is exactly like sparse_hash_map.h, with the word -// "sparse" replaced by "dense", except for the addition of -// set_empty_key(). -// -// YOU MUST CALL SET_EMPTY_KEY() IMMEDIATELY AFTER CONSTRUCTION. -// -// Otherwise your program will die in mysterious ways. (Note if you -// use the constructor that takes an InputIterator range, you pass in -// the empty key in the constructor, rather than after. As a result, -// this constructor differs from the standard STL version.) -// -// In other respects, we adhere mostly to the STL semantics for -// hash-map. One important exception is that insert() may invalidate -// iterators entirely -- STL semantics are that insert() may reorder -// iterators, but they all still refer to something valid in the -// hashtable. Not so for us. Likewise, insert() may invalidate -// pointers into the hashtable. (Whether insert invalidates iterators -// and pointers depends on whether it results in a hashtable resize). -// On the plus side, delete() doesn't invalidate iterators or pointers -// at all, or even change the ordering of elements. -// -// Here are a few "power user" tips: -// -// 1) set_deleted_key(): -// If you want to use erase() you *must* call set_deleted_key(), -// in addition to set_empty_key(), after construction. -// The deleted and empty keys must differ. -// -// 2) resize(0): -// When an item is deleted, its memory isn't freed right -// away. This allows you to iterate over a hashtable, -// and call erase(), without invalidating the iterator. -// To force the memory to be freed, call resize(0). -// For tr1 compatibility, this can also be called as rehash(0). -// -// 3) min_load_factor(0.0) -// Setting the minimum load factor to 0.0 guarantees that -// the hash table will never shrink. -// -// Roughly speaking: -// (1) dense_hash_map: fastest, uses the most memory unless entries are small -// (2) sparse_hash_map: slowest, uses the least memory -// (3) hash_map / unordered_map (STL): in the middle -// -// Typically I use sparse_hash_map when I care about space and/or when -// I need to save the hashtable on disk. I use hash_map otherwise. I -// don't personally use dense_hash_set ever; some people use it for -// small sets with lots of lookups. -// -// - dense_hash_map has, typically, about 78% memory overhead (if your -// data takes up X bytes, the hash_map uses .78X more bytes in overhead). -// - sparse_hash_map has about 4 bits overhead per entry. -// - sparse_hash_map can be 3-7 times slower than the others for lookup and, -// especially, inserts. See time_hash_map.cc for details. -// -// See /usr/(local/)?doc/sparsehash-*/dense_hash_map.html -// for information about how to use this class. - -#ifndef _DENSE_HASH_MAP_H_ -#define _DENSE_HASH_MAP_H_ - -#include <sparsehash/internal/sparseconfig.h> -#include <algorithm> // needed by stl_alloc -#include <functional> // for equal_to<>, select1st<>, etc -#include <memory> // for alloc -#include <utility> // for pair<> -#include <sparsehash/internal/densehashtable.h> // IWYU pragma: export -#include <sparsehash/internal/libc_allocator_with_realloc.h> -#include HASH_FUN_H // for hash<> -_START_GOOGLE_NAMESPACE_ - -template <class Key, class T, - class HashFcn = SPARSEHASH_HASH<Key>, // defined in sparseconfig.h - class EqualKey = std::equal_to<Key>, - class Alloc = libc_allocator_with_realloc<std::pair<const Key, T> > > -class dense_hash_map { - private: - // Apparently select1st is not stl-standard, so we define our own - struct SelectKey { - typedef const Key& result_type; - const Key& operator()(const std::pair<const Key, T>& p) const { - return p.first; - } - }; - struct SetKey { - void operator()(std::pair<const Key, T>* value, const Key& new_key) const { - *const_cast<Key*>(&value->first) = new_key; - // It would be nice to clear the rest of value here as well, in - // case it's taking up a lot of memory. We do this by clearing - // the value. This assumes T has a zero-arg constructor! - value->second = T(); - } - }; - // For operator[]. - struct DefaultValue { - std::pair<const Key, T> operator()(const Key& key) { - return std::make_pair(key, T()); - } - }; - - // The actual data - typedef dense_hashtable<std::pair<const Key, T>, Key, HashFcn, SelectKey, - SetKey, EqualKey, Alloc> ht; - ht rep; - - public: - typedef typename ht::key_type key_type; - typedef T data_type; - typedef T mapped_type; - typedef typename ht::value_type value_type; - typedef typename ht::hasher hasher; - typedef typename ht::key_equal key_equal; - typedef Alloc allocator_type; - - typedef typename ht::size_type size_type; - typedef typename ht::difference_type difference_type; - typedef typename ht::pointer pointer; - typedef typename ht::const_pointer const_pointer; - typedef typename ht::reference reference; - typedef typename ht::const_reference const_reference; - - typedef typename ht::iterator iterator; - typedef typename ht::const_iterator const_iterator; - typedef typename ht::local_iterator local_iterator; - typedef typename ht::const_local_iterator const_local_iterator; - - // Iterator functions - iterator begin() { return rep.begin(); } - iterator end() { return rep.end(); } - const_iterator begin() const { return rep.begin(); } - const_iterator end() const { return rep.end(); } - - - // These come from tr1's unordered_map. For us, a bucket has 0 or 1 elements. - local_iterator begin(size_type i) { return rep.begin(i); } - local_iterator end(size_type i) { return rep.end(i); } - const_local_iterator begin(size_type i) const { return rep.begin(i); } - const_local_iterator end(size_type i) const { return rep.end(i); } - - // Accessor functions - allocator_type get_allocator() const { return rep.get_allocator(); } - hasher hash_funct() const { return rep.hash_funct(); } - hasher hash_function() const { return hash_funct(); } - key_equal key_eq() const { return rep.key_eq(); } - - - // Constructors - explicit dense_hash_map(size_type expected_max_items_in_table = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { - } - - template <class InputIterator> - dense_hash_map(InputIterator f, InputIterator l, - const key_type& empty_key_val, - size_type expected_max_items_in_table = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { - set_empty_key(empty_key_val); - rep.insert(f, l); - } - // We use the default copy constructor - // We use the default operator=() - // We use the default destructor - - void clear() { rep.clear(); } - // This clears the hash map without resizing it down to the minimum - // bucket count, but rather keeps the number of buckets constant - void clear_no_resize() { rep.clear_no_resize(); } - void swap(dense_hash_map& hs) { rep.swap(hs.rep); } - - - // Functions concerning size - size_type size() const { return rep.size(); } - size_type max_size() const { return rep.max_size(); } - bool empty() const { return rep.empty(); } - size_type bucket_count() const { return rep.bucket_count(); } - size_type max_bucket_count() const { return rep.max_bucket_count(); } - - // These are tr1 methods. bucket() is the bucket the key is or would be in. - size_type bucket_size(size_type i) const { return rep.bucket_size(i); } - size_type bucket(const key_type& key) const { return rep.bucket(key); } - float load_factor() const { - return size() * 1.0f / bucket_count(); - } - float max_load_factor() const { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - return grow; - } - void max_load_factor(float new_grow) { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - rep.set_resizing_parameters(shrink, new_grow); - } - // These aren't tr1 methods but perhaps ought to be. - float min_load_factor() const { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - return shrink; - } - void min_load_factor(float new_shrink) { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - rep.set_resizing_parameters(new_shrink, grow); - } - // Deprecated; use min_load_factor() or max_load_factor() instead. - void set_resizing_parameters(float shrink, float grow) { - rep.set_resizing_parameters(shrink, grow); - } - - void resize(size_type hint) { rep.resize(hint); } - void rehash(size_type hint) { resize(hint); } // the tr1 name - - // Lookup routines - iterator find(const key_type& key) { return rep.find(key); } - const_iterator find(const key_type& key) const { return rep.find(key); } - - data_type& operator[](const key_type& key) { // This is our value-add! - // If key is in the hashtable, returns find(key)->second, - // otherwise returns insert(value_type(key, T()).first->second. - // Note it does not create an empty T unless the find fails. - return rep.template find_or_insert<DefaultValue>(key).second; - } - - size_type count(const key_type& key) const { return rep.count(key); } - - std::pair<iterator, iterator> equal_range(const key_type& key) { - return rep.equal_range(key); - } - std::pair<const_iterator, const_iterator> equal_range(const key_type& key) - const { - return rep.equal_range(key); - } - - - // Insertion routines - std::pair<iterator, bool> insert(const value_type& obj) { - return rep.insert(obj); - } - template <class InputIterator> void insert(InputIterator f, InputIterator l) { - rep.insert(f, l); - } - void insert(const_iterator f, const_iterator l) { - rep.insert(f, l); - } - // Required for std::insert_iterator; the passed-in iterator is ignored. - iterator insert(iterator, const value_type& obj) { - return insert(obj).first; - } - - // Deletion and empty routines - // THESE ARE NON-STANDARD! I make you specify an "impossible" key - // value to identify deleted and empty buckets. You can change the - // deleted key as time goes on, or get rid of it entirely to be insert-only. - void set_empty_key(const key_type& key) { // YOU MUST CALL THIS! - rep.set_empty_key(value_type(key, data_type())); // rep wants a value - } - key_type empty_key() const { - return rep.empty_key().first; // rep returns a value - } - - void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); } - void clear_deleted_key() { rep.clear_deleted_key(); } - key_type deleted_key() const { return rep.deleted_key(); } - - // These are standard - size_type erase(const key_type& key) { return rep.erase(key); } - void erase(iterator it) { rep.erase(it); } - void erase(iterator f, iterator l) { rep.erase(f, l); } - - - // Comparison - bool operator==(const dense_hash_map& hs) const { return rep == hs.rep; } - bool operator!=(const dense_hash_map& hs) const { return rep != hs.rep; } - - - // I/O -- this is an add-on for writing hash map to disk - // - // For maximum flexibility, this does not assume a particular - // file type (though it will probably be a FILE *). We just pass - // the fp through to rep. - - // If your keys and values are simple enough, you can pass this - // serializer to serialize()/unserialize(). "Simple enough" means - // value_type is a POD type that contains no pointers. Note, - // however, we don't try to normalize endianness. - typedef typename ht::NopointerSerializer NopointerSerializer; - - // serializer: a class providing operator()(OUTPUT*, const value_type&) - // (writing value_type to OUTPUT). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a - // pointer to a class providing size_t Write(const void*, size_t), - // which writes a buffer into a stream (which fp presumably - // owns) and returns the number of bytes successfully written. - // Note basic_ostream<not_char> is not currently supported. - template <typename ValueSerializer, typename OUTPUT> - bool serialize(ValueSerializer serializer, OUTPUT* fp) { - return rep.serialize(serializer, fp); - } - - // serializer: a functor providing operator()(INPUT*, value_type*) - // (reading from INPUT and into value_type). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a - // pointer to a class providing size_t Read(void*, size_t), - // which reads into a buffer from a stream (which fp presumably - // owns) and returns the number of bytes successfully read. - // Note basic_istream<not_char> is not currently supported. - // NOTE: Since value_type is std::pair<const Key, T>, ValueSerializer - // may need to do a const cast in order to fill in the key. - template <typename ValueSerializer, typename INPUT> - bool unserialize(ValueSerializer serializer, INPUT* fp) { - return rep.unserialize(serializer, fp); - } -}; - -// We need a global swap as well -template <class Key, class T, class HashFcn, class EqualKey, class Alloc> -inline void swap(dense_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm1, - dense_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm2) { - hm1.swap(hm2); -} - -_END_GOOGLE_NAMESPACE_ - -#endif /* _DENSE_HASH_MAP_H_ */ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ---- +// +// This is just a very thin wrapper over densehashtable.h, just +// like sgi stl's stl_hash_map is a very thin wrapper over +// stl_hashtable. The major thing we define is operator[], because +// we have a concept of a data_type which stl_hashtable doesn't +// (it only has a key and a value). +// +// NOTE: this is exactly like sparse_hash_map.h, with the word +// "sparse" replaced by "dense", except for the addition of +// set_empty_key(). +// +// YOU MUST CALL SET_EMPTY_KEY() IMMEDIATELY AFTER CONSTRUCTION. +// +// Otherwise your program will die in mysterious ways. (Note if you +// use the constructor that takes an InputIterator range, you pass in +// the empty key in the constructor, rather than after. As a result, +// this constructor differs from the standard STL version.) +// +// In other respects, we adhere mostly to the STL semantics for +// hash-map. One important exception is that insert() may invalidate +// iterators entirely -- STL semantics are that insert() may reorder +// iterators, but they all still refer to something valid in the +// hashtable. Not so for us. Likewise, insert() may invalidate +// pointers into the hashtable. (Whether insert invalidates iterators +// and pointers depends on whether it results in a hashtable resize). +// On the plus side, delete() doesn't invalidate iterators or pointers +// at all, or even change the ordering of elements. +// +// Here are a few "power user" tips: +// +// 1) set_deleted_key(): +// If you want to use erase() you *must* call set_deleted_key(), +// in addition to set_empty_key(), after construction. +// The deleted and empty keys must differ. +// +// 2) resize(0): +// When an item is deleted, its memory isn't freed right +// away. This allows you to iterate over a hashtable, +// and call erase(), without invalidating the iterator. +// To force the memory to be freed, call resize(0). +// For tr1 compatibility, this can also be called as rehash(0). +// +// 3) min_load_factor(0.0) +// Setting the minimum load factor to 0.0 guarantees that +// the hash table will never shrink. +// +// Roughly speaking: +// (1) dense_hash_map: fastest, uses the most memory unless entries are small +// (2) sparse_hash_map: slowest, uses the least memory +// (3) hash_map / unordered_map (STL): in the middle +// +// Typically I use sparse_hash_map when I care about space and/or when +// I need to save the hashtable on disk. I use hash_map otherwise. I +// don't personally use dense_hash_set ever; some people use it for +// small sets with lots of lookups. +// +// - dense_hash_map has, typically, about 78% memory overhead (if your +// data takes up X bytes, the hash_map uses .78X more bytes in overhead). +// - sparse_hash_map has about 4 bits overhead per entry. +// - sparse_hash_map can be 3-7 times slower than the others for lookup and, +// especially, inserts. See time_hash_map.cc for details. +// +// See /usr/(local/)?doc/sparsehash-*/dense_hash_map.html +// for information about how to use this class. + +#ifndef _DENSE_HASH_MAP_H_ +#define _DENSE_HASH_MAP_H_ + +#include <sparsehash/internal/sparseconfig.h> +#include <algorithm> // needed by stl_alloc +#include <functional> // for equal_to<>, select1st<>, etc +#include <memory> // for alloc +#include <utility> // for pair<> +#include <sparsehash/internal/densehashtable.h> // IWYU pragma: export +#include <sparsehash/internal/libc_allocator_with_realloc.h> +#include HASH_FUN_H // for hash<> +_START_GOOGLE_NAMESPACE_ + +template <class Key, class T, + class HashFcn = SPARSEHASH_HASH<Key>, // defined in sparseconfig.h + class EqualKey = std::equal_to<Key>, + class Alloc = libc_allocator_with_realloc<std::pair<const Key, T> > > +class dense_hash_map { + private: + // Apparently select1st is not stl-standard, so we define our own + struct SelectKey { + typedef const Key& result_type; + const Key& operator()(const std::pair<const Key, T>& p) const { + return p.first; + } + }; + struct SetKey { + void operator()(std::pair<const Key, T>* value, const Key& new_key) const { + *const_cast<Key*>(&value->first) = new_key; + // It would be nice to clear the rest of value here as well, in + // case it's taking up a lot of memory. We do this by clearing + // the value. This assumes T has a zero-arg constructor! + value->second = T(); + } + }; + // For operator[]. + struct DefaultValue { + std::pair<const Key, T> operator()(const Key& key) { + return std::make_pair(key, T()); + } + }; + + // The actual data + typedef dense_hashtable<std::pair<const Key, T>, Key, HashFcn, SelectKey, + SetKey, EqualKey, Alloc> ht; + ht rep; + + public: + typedef typename ht::key_type key_type; + typedef T data_type; + typedef T mapped_type; + typedef typename ht::value_type value_type; + typedef typename ht::hasher hasher; + typedef typename ht::key_equal key_equal; + typedef Alloc allocator_type; + + typedef typename ht::size_type size_type; + typedef typename ht::difference_type difference_type; + typedef typename ht::pointer pointer; + typedef typename ht::const_pointer const_pointer; + typedef typename ht::reference reference; + typedef typename ht::const_reference const_reference; + + typedef typename ht::iterator iterator; + typedef typename ht::const_iterator const_iterator; + typedef typename ht::local_iterator local_iterator; + typedef typename ht::const_local_iterator const_local_iterator; + + // Iterator functions + iterator begin() { return rep.begin(); } + iterator end() { return rep.end(); } + const_iterator begin() const { return rep.begin(); } + const_iterator end() const { return rep.end(); } + + + // These come from tr1's unordered_map. For us, a bucket has 0 or 1 elements. + local_iterator begin(size_type i) { return rep.begin(i); } + local_iterator end(size_type i) { return rep.end(i); } + const_local_iterator begin(size_type i) const { return rep.begin(i); } + const_local_iterator end(size_type i) const { return rep.end(i); } + + // Accessor functions + allocator_type get_allocator() const { return rep.get_allocator(); } + hasher hash_funct() const { return rep.hash_funct(); } + hasher hash_function() const { return hash_funct(); } + key_equal key_eq() const { return rep.key_eq(); } + + + // Constructors + explicit dense_hash_map(size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { + } + + template <class InputIterator> + dense_hash_map(InputIterator f, InputIterator l, + const key_type& empty_key_val, + size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { + set_empty_key(empty_key_val); + rep.insert(f, l); + } + // We use the default copy constructor + // We use the default operator=() + // We use the default destructor + + void clear() { rep.clear(); } + // This clears the hash map without resizing it down to the minimum + // bucket count, but rather keeps the number of buckets constant + void clear_no_resize() { rep.clear_no_resize(); } + void swap(dense_hash_map& hs) { rep.swap(hs.rep); } + + + // Functions concerning size + size_type size() const { return rep.size(); } + size_type max_size() const { return rep.max_size(); } + bool empty() const { return rep.empty(); } + size_type bucket_count() const { return rep.bucket_count(); } + size_type max_bucket_count() const { return rep.max_bucket_count(); } + + // These are tr1 methods. bucket() is the bucket the key is or would be in. + size_type bucket_size(size_type i) const { return rep.bucket_size(i); } + size_type bucket(const key_type& key) const { return rep.bucket(key); } + float load_factor() const { + return size() * 1.0f / bucket_count(); + } + float max_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return grow; + } + void max_load_factor(float new_grow) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(shrink, new_grow); + } + // These aren't tr1 methods but perhaps ought to be. + float min_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return shrink; + } + void min_load_factor(float new_shrink) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(new_shrink, grow); + } + // Deprecated; use min_load_factor() or max_load_factor() instead. + void set_resizing_parameters(float shrink, float grow) { + rep.set_resizing_parameters(shrink, grow); + } + + void resize(size_type hint) { rep.resize(hint); } + void rehash(size_type hint) { resize(hint); } // the tr1 name + + // Lookup routines + iterator find(const key_type& key) { return rep.find(key); } + const_iterator find(const key_type& key) const { return rep.find(key); } + + data_type& operator[](const key_type& key) { // This is our value-add! + // If key is in the hashtable, returns find(key)->second, + // otherwise returns insert(value_type(key, T()).first->second. + // Note it does not create an empty T unless the find fails. + return rep.template find_or_insert<DefaultValue>(key).second; + } + + size_type count(const key_type& key) const { return rep.count(key); } + + std::pair<iterator, iterator> equal_range(const key_type& key) { + return rep.equal_range(key); + } + std::pair<const_iterator, const_iterator> equal_range(const key_type& key) + const { + return rep.equal_range(key); + } + + + // Insertion routines + std::pair<iterator, bool> insert(const value_type& obj) { + return rep.insert(obj); + } + template <class InputIterator> void insert(InputIterator f, InputIterator l) { + rep.insert(f, l); + } + void insert(const_iterator f, const_iterator l) { + rep.insert(f, l); + } + // Required for std::insert_iterator; the passed-in iterator is ignored. + iterator insert(iterator, const value_type& obj) { + return insert(obj).first; + } + + // Deletion and empty routines + // THESE ARE NON-STANDARD! I make you specify an "impossible" key + // value to identify deleted and empty buckets. You can change the + // deleted key as time goes on, or get rid of it entirely to be insert-only. + void set_empty_key(const key_type& key) { // YOU MUST CALL THIS! + rep.set_empty_key(value_type(key, data_type())); // rep wants a value + } + key_type empty_key() const { + return rep.empty_key().first; // rep returns a value + } + + void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); } + void clear_deleted_key() { rep.clear_deleted_key(); } + key_type deleted_key() const { return rep.deleted_key(); } + + // These are standard + size_type erase(const key_type& key) { return rep.erase(key); } + void erase(iterator it) { rep.erase(it); } + void erase(iterator f, iterator l) { rep.erase(f, l); } + + + // Comparison + bool operator==(const dense_hash_map& hs) const { return rep == hs.rep; } + bool operator!=(const dense_hash_map& hs) const { return rep != hs.rep; } + + + // I/O -- this is an add-on for writing hash map to disk + // + // For maximum flexibility, this does not assume a particular + // file type (though it will probably be a FILE *). We just pass + // the fp through to rep. + + // If your keys and values are simple enough, you can pass this + // serializer to serialize()/unserialize(). "Simple enough" means + // value_type is a POD type that contains no pointers. Note, + // however, we don't try to normalize endianness. + typedef typename ht::NopointerSerializer NopointerSerializer; + + // serializer: a class providing operator()(OUTPUT*, const value_type&) + // (writing value_type to OUTPUT). You can specify a + // NopointerSerializer object if appropriate (see above). + // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a + // pointer to a class providing size_t Write(const void*, size_t), + // which writes a buffer into a stream (which fp presumably + // owns) and returns the number of bytes successfully written. + // Note basic_ostream<not_char> is not currently supported. + template <typename ValueSerializer, typename OUTPUT> + bool serialize(ValueSerializer serializer, OUTPUT* fp) { + return rep.serialize(serializer, fp); + } + + // serializer: a functor providing operator()(INPUT*, value_type*) + // (reading from INPUT and into value_type). You can specify a + // NopointerSerializer object if appropriate (see above). + // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a + // pointer to a class providing size_t Read(void*, size_t), + // which reads into a buffer from a stream (which fp presumably + // owns) and returns the number of bytes successfully read. + // Note basic_istream<not_char> is not currently supported. + // NOTE: Since value_type is std::pair<const Key, T>, ValueSerializer + // may need to do a const cast in order to fill in the key. + template <typename ValueSerializer, typename INPUT> + bool unserialize(ValueSerializer serializer, INPUT* fp) { + return rep.unserialize(serializer, fp); + } +}; + +// We need a global swap as well +template <class Key, class T, class HashFcn, class EqualKey, class Alloc> +inline void swap(dense_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm1, + dense_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm2) { + hm1.swap(hm2); +} + +_END_GOOGLE_NAMESPACE_ + +#endif /* _DENSE_HASH_MAP_H_ */ diff --git a/contrib/libs/sparsehash/src/sparsehash/dense_hash_set b/contrib/libs/sparsehash/src/sparsehash/dense_hash_set index 5238cd1e86..050b15d1d5 100644 --- a/contrib/libs/sparsehash/src/sparsehash/dense_hash_set +++ b/contrib/libs/sparsehash/src/sparsehash/dense_hash_set @@ -1,338 +1,338 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// -// This is just a very thin wrapper over densehashtable.h, just -// like sgi stl's stl_hash_set is a very thin wrapper over -// stl_hashtable. The major thing we define is operator[], because -// we have a concept of a data_type which stl_hashtable doesn't -// (it only has a key and a value). -// -// This is more different from dense_hash_map than you might think, -// because all iterators for sets are const (you obviously can't -// change the key, and for sets there is no value). -// -// NOTE: this is exactly like sparse_hash_set.h, with the word -// "sparse" replaced by "dense", except for the addition of -// set_empty_key(). -// -// YOU MUST CALL SET_EMPTY_KEY() IMMEDIATELY AFTER CONSTRUCTION. -// -// Otherwise your program will die in mysterious ways. (Note if you -// use the constructor that takes an InputIterator range, you pass in -// the empty key in the constructor, rather than after. As a result, -// this constructor differs from the standard STL version.) -// -// In other respects, we adhere mostly to the STL semantics for -// hash-map. One important exception is that insert() may invalidate -// iterators entirely -- STL semantics are that insert() may reorder -// iterators, but they all still refer to something valid in the -// hashtable. Not so for us. Likewise, insert() may invalidate -// pointers into the hashtable. (Whether insert invalidates iterators -// and pointers depends on whether it results in a hashtable resize). -// On the plus side, delete() doesn't invalidate iterators or pointers -// at all, or even change the ordering of elements. -// -// Here are a few "power user" tips: -// -// 1) set_deleted_key(): -// If you want to use erase() you must call set_deleted_key(), -// in addition to set_empty_key(), after construction. -// The deleted and empty keys must differ. -// -// 2) resize(0): -// When an item is deleted, its memory isn't freed right -// away. This allows you to iterate over a hashtable, -// and call erase(), without invalidating the iterator. -// To force the memory to be freed, call resize(0). -// For tr1 compatibility, this can also be called as rehash(0). -// -// 3) min_load_factor(0.0) -// Setting the minimum load factor to 0.0 guarantees that -// the hash table will never shrink. -// -// Roughly speaking: -// (1) dense_hash_set: fastest, uses the most memory unless entries are small -// (2) sparse_hash_set: slowest, uses the least memory -// (3) hash_set / unordered_set (STL): in the middle -// -// Typically I use sparse_hash_set when I care about space and/or when -// I need to save the hashtable on disk. I use hash_set otherwise. I -// don't personally use dense_hash_set ever; some people use it for -// small sets with lots of lookups. -// -// - dense_hash_set has, typically, about 78% memory overhead (if your -// data takes up X bytes, the hash_set uses .78X more bytes in overhead). -// - sparse_hash_set has about 4 bits overhead per entry. -// - sparse_hash_set can be 3-7 times slower than the others for lookup and, -// especially, inserts. See time_hash_map.cc for details. -// -// See /usr/(local/)?doc/sparsehash-*/dense_hash_set.html -// for information about how to use this class. - -#ifndef _DENSE_HASH_SET_H_ -#define _DENSE_HASH_SET_H_ - -#include <sparsehash/internal/sparseconfig.h> -#include <algorithm> // needed by stl_alloc -#include <functional> // for equal_to<>, select1st<>, etc -#include <memory> // for alloc -#include <utility> // for pair<> -#include <sparsehash/internal/densehashtable.h> // IWYU pragma: export -#include <sparsehash/internal/libc_allocator_with_realloc.h> -#include HASH_FUN_H // for hash<> -_START_GOOGLE_NAMESPACE_ - -template <class Value, - class HashFcn = SPARSEHASH_HASH<Value>, // defined in sparseconfig.h - class EqualKey = std::equal_to<Value>, - class Alloc = libc_allocator_with_realloc<Value> > -class dense_hash_set { - private: - // Apparently identity is not stl-standard, so we define our own - struct Identity { - typedef const Value& result_type; - const Value& operator()(const Value& v) const { return v; } - }; - struct SetKey { - void operator()(Value* value, const Value& new_key) const { - *value = new_key; - } - }; - - // The actual data - typedef dense_hashtable<Value, Value, HashFcn, Identity, SetKey, - EqualKey, Alloc> ht; - ht rep; - - public: - typedef typename ht::key_type key_type; - typedef typename ht::value_type value_type; - typedef typename ht::hasher hasher; - typedef typename ht::key_equal key_equal; - typedef Alloc allocator_type; - - typedef typename ht::size_type size_type; - typedef typename ht::difference_type difference_type; - typedef typename ht::const_pointer pointer; - typedef typename ht::const_pointer const_pointer; - typedef typename ht::const_reference reference; - typedef typename ht::const_reference const_reference; - - typedef typename ht::const_iterator iterator; - typedef typename ht::const_iterator const_iterator; - typedef typename ht::const_local_iterator local_iterator; - typedef typename ht::const_local_iterator const_local_iterator; - - - // Iterator functions -- recall all iterators are const - iterator begin() const { return rep.begin(); } - iterator end() const { return rep.end(); } - - // These come from tr1's unordered_set. For us, a bucket has 0 or 1 elements. - local_iterator begin(size_type i) const { return rep.begin(i); } - local_iterator end(size_type i) const { return rep.end(i); } - - - // Accessor functions - allocator_type get_allocator() const { return rep.get_allocator(); } - hasher hash_funct() const { return rep.hash_funct(); } - hasher hash_function() const { return hash_funct(); } // tr1 name - key_equal key_eq() const { return rep.key_eq(); } - - - // Constructors - explicit dense_hash_set(size_type expected_max_items_in_table = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { - } - - template <class InputIterator> - dense_hash_set(InputIterator f, InputIterator l, - const key_type& empty_key_val, - size_type expected_max_items_in_table = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { - set_empty_key(empty_key_val); - rep.insert(f, l); - } - // We use the default copy constructor - // We use the default operator=() - // We use the default destructor - - void clear() { rep.clear(); } - // This clears the hash set without resizing it down to the minimum - // bucket count, but rather keeps the number of buckets constant - void clear_no_resize() { rep.clear_no_resize(); } - void swap(dense_hash_set& hs) { rep.swap(hs.rep); } - - - // Functions concerning size - size_type size() const { return rep.size(); } - size_type max_size() const { return rep.max_size(); } - bool empty() const { return rep.empty(); } - size_type bucket_count() const { return rep.bucket_count(); } - size_type max_bucket_count() const { return rep.max_bucket_count(); } - - // These are tr1 methods. bucket() is the bucket the key is or would be in. - size_type bucket_size(size_type i) const { return rep.bucket_size(i); } - size_type bucket(const key_type& key) const { return rep.bucket(key); } - float load_factor() const { - return size() * 1.0f / bucket_count(); - } - float max_load_factor() const { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - return grow; - } - void max_load_factor(float new_grow) { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - rep.set_resizing_parameters(shrink, new_grow); - } - // These aren't tr1 methods but perhaps ought to be. - float min_load_factor() const { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - return shrink; - } - void min_load_factor(float new_shrink) { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - rep.set_resizing_parameters(new_shrink, grow); - } - // Deprecated; use min_load_factor() or max_load_factor() instead. - void set_resizing_parameters(float shrink, float grow) { - rep.set_resizing_parameters(shrink, grow); - } - - void resize(size_type hint) { rep.resize(hint); } - void rehash(size_type hint) { resize(hint); } // the tr1 name - - // Lookup routines - iterator find(const key_type& key) const { return rep.find(key); } - - size_type count(const key_type& key) const { return rep.count(key); } - - std::pair<iterator, iterator> equal_range(const key_type& key) const { - return rep.equal_range(key); - } - - - // Insertion routines - std::pair<iterator, bool> insert(const value_type& obj) { - std::pair<typename ht::iterator, bool> p = rep.insert(obj); - return std::pair<iterator, bool>(p.first, p.second); // const to non-const - } - template <class InputIterator> void insert(InputIterator f, InputIterator l) { - rep.insert(f, l); - } - void insert(const_iterator f, const_iterator l) { - rep.insert(f, l); - } - // Required for std::insert_iterator; the passed-in iterator is ignored. - iterator insert(iterator, const value_type& obj) { - return insert(obj).first; - } - - // Deletion and empty routines - // THESE ARE NON-STANDARD! I make you specify an "impossible" key - // value to identify deleted and empty buckets. You can change the - // deleted key as time goes on, or get rid of it entirely to be insert-only. - void set_empty_key(const key_type& key) { rep.set_empty_key(key); } - key_type empty_key() const { return rep.empty_key(); } - - void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); } - void clear_deleted_key() { rep.clear_deleted_key(); } - key_type deleted_key() const { return rep.deleted_key(); } - - // These are standard - size_type erase(const key_type& key) { return rep.erase(key); } - void erase(iterator it) { rep.erase(it); } - void erase(iterator f, iterator l) { rep.erase(f, l); } - - - // Comparison - bool operator==(const dense_hash_set& hs) const { return rep == hs.rep; } - bool operator!=(const dense_hash_set& hs) const { return rep != hs.rep; } - - - // I/O -- this is an add-on for writing metainformation to disk - // - // For maximum flexibility, this does not assume a particular - // file type (though it will probably be a FILE *). We just pass - // the fp through to rep. - - // If your keys and values are simple enough, you can pass this - // serializer to serialize()/unserialize(). "Simple enough" means - // value_type is a POD type that contains no pointers. Note, - // however, we don't try to normalize endianness. - typedef typename ht::NopointerSerializer NopointerSerializer; - - // serializer: a class providing operator()(OUTPUT*, const value_type&) - // (writing value_type to OUTPUT). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a - // pointer to a class providing size_t Write(const void*, size_t), - // which writes a buffer into a stream (which fp presumably - // owns) and returns the number of bytes successfully written. - // Note basic_ostream<not_char> is not currently supported. - template <typename ValueSerializer, typename OUTPUT> - bool serialize(ValueSerializer serializer, OUTPUT* fp) { - return rep.serialize(serializer, fp); - } - - // serializer: a functor providing operator()(INPUT*, value_type*) - // (reading from INPUT and into value_type). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a - // pointer to a class providing size_t Read(void*, size_t), - // which reads into a buffer from a stream (which fp presumably - // owns) and returns the number of bytes successfully read. - // Note basic_istream<not_char> is not currently supported. - template <typename ValueSerializer, typename INPUT> - bool unserialize(ValueSerializer serializer, INPUT* fp) { - return rep.unserialize(serializer, fp); - } -}; - -template <class Val, class HashFcn, class EqualKey, class Alloc> -inline void swap(dense_hash_set<Val, HashFcn, EqualKey, Alloc>& hs1, - dense_hash_set<Val, HashFcn, EqualKey, Alloc>& hs2) { - hs1.swap(hs2); -} - -_END_GOOGLE_NAMESPACE_ - -#endif /* _DENSE_HASH_SET_H_ */ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// +// This is just a very thin wrapper over densehashtable.h, just +// like sgi stl's stl_hash_set is a very thin wrapper over +// stl_hashtable. The major thing we define is operator[], because +// we have a concept of a data_type which stl_hashtable doesn't +// (it only has a key and a value). +// +// This is more different from dense_hash_map than you might think, +// because all iterators for sets are const (you obviously can't +// change the key, and for sets there is no value). +// +// NOTE: this is exactly like sparse_hash_set.h, with the word +// "sparse" replaced by "dense", except for the addition of +// set_empty_key(). +// +// YOU MUST CALL SET_EMPTY_KEY() IMMEDIATELY AFTER CONSTRUCTION. +// +// Otherwise your program will die in mysterious ways. (Note if you +// use the constructor that takes an InputIterator range, you pass in +// the empty key in the constructor, rather than after. As a result, +// this constructor differs from the standard STL version.) +// +// In other respects, we adhere mostly to the STL semantics for +// hash-map. One important exception is that insert() may invalidate +// iterators entirely -- STL semantics are that insert() may reorder +// iterators, but they all still refer to something valid in the +// hashtable. Not so for us. Likewise, insert() may invalidate +// pointers into the hashtable. (Whether insert invalidates iterators +// and pointers depends on whether it results in a hashtable resize). +// On the plus side, delete() doesn't invalidate iterators or pointers +// at all, or even change the ordering of elements. +// +// Here are a few "power user" tips: +// +// 1) set_deleted_key(): +// If you want to use erase() you must call set_deleted_key(), +// in addition to set_empty_key(), after construction. +// The deleted and empty keys must differ. +// +// 2) resize(0): +// When an item is deleted, its memory isn't freed right +// away. This allows you to iterate over a hashtable, +// and call erase(), without invalidating the iterator. +// To force the memory to be freed, call resize(0). +// For tr1 compatibility, this can also be called as rehash(0). +// +// 3) min_load_factor(0.0) +// Setting the minimum load factor to 0.0 guarantees that +// the hash table will never shrink. +// +// Roughly speaking: +// (1) dense_hash_set: fastest, uses the most memory unless entries are small +// (2) sparse_hash_set: slowest, uses the least memory +// (3) hash_set / unordered_set (STL): in the middle +// +// Typically I use sparse_hash_set when I care about space and/or when +// I need to save the hashtable on disk. I use hash_set otherwise. I +// don't personally use dense_hash_set ever; some people use it for +// small sets with lots of lookups. +// +// - dense_hash_set has, typically, about 78% memory overhead (if your +// data takes up X bytes, the hash_set uses .78X more bytes in overhead). +// - sparse_hash_set has about 4 bits overhead per entry. +// - sparse_hash_set can be 3-7 times slower than the others for lookup and, +// especially, inserts. See time_hash_map.cc for details. +// +// See /usr/(local/)?doc/sparsehash-*/dense_hash_set.html +// for information about how to use this class. + +#ifndef _DENSE_HASH_SET_H_ +#define _DENSE_HASH_SET_H_ + +#include <sparsehash/internal/sparseconfig.h> +#include <algorithm> // needed by stl_alloc +#include <functional> // for equal_to<>, select1st<>, etc +#include <memory> // for alloc +#include <utility> // for pair<> +#include <sparsehash/internal/densehashtable.h> // IWYU pragma: export +#include <sparsehash/internal/libc_allocator_with_realloc.h> +#include HASH_FUN_H // for hash<> +_START_GOOGLE_NAMESPACE_ + +template <class Value, + class HashFcn = SPARSEHASH_HASH<Value>, // defined in sparseconfig.h + class EqualKey = std::equal_to<Value>, + class Alloc = libc_allocator_with_realloc<Value> > +class dense_hash_set { + private: + // Apparently identity is not stl-standard, so we define our own + struct Identity { + typedef const Value& result_type; + const Value& operator()(const Value& v) const { return v; } + }; + struct SetKey { + void operator()(Value* value, const Value& new_key) const { + *value = new_key; + } + }; + + // The actual data + typedef dense_hashtable<Value, Value, HashFcn, Identity, SetKey, + EqualKey, Alloc> ht; + ht rep; + + public: + typedef typename ht::key_type key_type; + typedef typename ht::value_type value_type; + typedef typename ht::hasher hasher; + typedef typename ht::key_equal key_equal; + typedef Alloc allocator_type; + + typedef typename ht::size_type size_type; + typedef typename ht::difference_type difference_type; + typedef typename ht::const_pointer pointer; + typedef typename ht::const_pointer const_pointer; + typedef typename ht::const_reference reference; + typedef typename ht::const_reference const_reference; + + typedef typename ht::const_iterator iterator; + typedef typename ht::const_iterator const_iterator; + typedef typename ht::const_local_iterator local_iterator; + typedef typename ht::const_local_iterator const_local_iterator; + + + // Iterator functions -- recall all iterators are const + iterator begin() const { return rep.begin(); } + iterator end() const { return rep.end(); } + + // These come from tr1's unordered_set. For us, a bucket has 0 or 1 elements. + local_iterator begin(size_type i) const { return rep.begin(i); } + local_iterator end(size_type i) const { return rep.end(i); } + + + // Accessor functions + allocator_type get_allocator() const { return rep.get_allocator(); } + hasher hash_funct() const { return rep.hash_funct(); } + hasher hash_function() const { return hash_funct(); } // tr1 name + key_equal key_eq() const { return rep.key_eq(); } + + + // Constructors + explicit dense_hash_set(size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { + } + + template <class InputIterator> + dense_hash_set(InputIterator f, InputIterator l, + const key_type& empty_key_val, + size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { + set_empty_key(empty_key_val); + rep.insert(f, l); + } + // We use the default copy constructor + // We use the default operator=() + // We use the default destructor + + void clear() { rep.clear(); } + // This clears the hash set without resizing it down to the minimum + // bucket count, but rather keeps the number of buckets constant + void clear_no_resize() { rep.clear_no_resize(); } + void swap(dense_hash_set& hs) { rep.swap(hs.rep); } + + + // Functions concerning size + size_type size() const { return rep.size(); } + size_type max_size() const { return rep.max_size(); } + bool empty() const { return rep.empty(); } + size_type bucket_count() const { return rep.bucket_count(); } + size_type max_bucket_count() const { return rep.max_bucket_count(); } + + // These are tr1 methods. bucket() is the bucket the key is or would be in. + size_type bucket_size(size_type i) const { return rep.bucket_size(i); } + size_type bucket(const key_type& key) const { return rep.bucket(key); } + float load_factor() const { + return size() * 1.0f / bucket_count(); + } + float max_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return grow; + } + void max_load_factor(float new_grow) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(shrink, new_grow); + } + // These aren't tr1 methods but perhaps ought to be. + float min_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return shrink; + } + void min_load_factor(float new_shrink) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(new_shrink, grow); + } + // Deprecated; use min_load_factor() or max_load_factor() instead. + void set_resizing_parameters(float shrink, float grow) { + rep.set_resizing_parameters(shrink, grow); + } + + void resize(size_type hint) { rep.resize(hint); } + void rehash(size_type hint) { resize(hint); } // the tr1 name + + // Lookup routines + iterator find(const key_type& key) const { return rep.find(key); } + + size_type count(const key_type& key) const { return rep.count(key); } + + std::pair<iterator, iterator> equal_range(const key_type& key) const { + return rep.equal_range(key); + } + + + // Insertion routines + std::pair<iterator, bool> insert(const value_type& obj) { + std::pair<typename ht::iterator, bool> p = rep.insert(obj); + return std::pair<iterator, bool>(p.first, p.second); // const to non-const + } + template <class InputIterator> void insert(InputIterator f, InputIterator l) { + rep.insert(f, l); + } + void insert(const_iterator f, const_iterator l) { + rep.insert(f, l); + } + // Required for std::insert_iterator; the passed-in iterator is ignored. + iterator insert(iterator, const value_type& obj) { + return insert(obj).first; + } + + // Deletion and empty routines + // THESE ARE NON-STANDARD! I make you specify an "impossible" key + // value to identify deleted and empty buckets. You can change the + // deleted key as time goes on, or get rid of it entirely to be insert-only. + void set_empty_key(const key_type& key) { rep.set_empty_key(key); } + key_type empty_key() const { return rep.empty_key(); } + + void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); } + void clear_deleted_key() { rep.clear_deleted_key(); } + key_type deleted_key() const { return rep.deleted_key(); } + + // These are standard + size_type erase(const key_type& key) { return rep.erase(key); } + void erase(iterator it) { rep.erase(it); } + void erase(iterator f, iterator l) { rep.erase(f, l); } + + + // Comparison + bool operator==(const dense_hash_set& hs) const { return rep == hs.rep; } + bool operator!=(const dense_hash_set& hs) const { return rep != hs.rep; } + + + // I/O -- this is an add-on for writing metainformation to disk + // + // For maximum flexibility, this does not assume a particular + // file type (though it will probably be a FILE *). We just pass + // the fp through to rep. + + // If your keys and values are simple enough, you can pass this + // serializer to serialize()/unserialize(). "Simple enough" means + // value_type is a POD type that contains no pointers. Note, + // however, we don't try to normalize endianness. + typedef typename ht::NopointerSerializer NopointerSerializer; + + // serializer: a class providing operator()(OUTPUT*, const value_type&) + // (writing value_type to OUTPUT). You can specify a + // NopointerSerializer object if appropriate (see above). + // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a + // pointer to a class providing size_t Write(const void*, size_t), + // which writes a buffer into a stream (which fp presumably + // owns) and returns the number of bytes successfully written. + // Note basic_ostream<not_char> is not currently supported. + template <typename ValueSerializer, typename OUTPUT> + bool serialize(ValueSerializer serializer, OUTPUT* fp) { + return rep.serialize(serializer, fp); + } + + // serializer: a functor providing operator()(INPUT*, value_type*) + // (reading from INPUT and into value_type). You can specify a + // NopointerSerializer object if appropriate (see above). + // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a + // pointer to a class providing size_t Read(void*, size_t), + // which reads into a buffer from a stream (which fp presumably + // owns) and returns the number of bytes successfully read. + // Note basic_istream<not_char> is not currently supported. + template <typename ValueSerializer, typename INPUT> + bool unserialize(ValueSerializer serializer, INPUT* fp) { + return rep.unserialize(serializer, fp); + } +}; + +template <class Val, class HashFcn, class EqualKey, class Alloc> +inline void swap(dense_hash_set<Val, HashFcn, EqualKey, Alloc>& hs1, + dense_hash_set<Val, HashFcn, EqualKey, Alloc>& hs2) { + hs1.swap(hs2); +} + +_END_GOOGLE_NAMESPACE_ + +#endif /* _DENSE_HASH_SET_H_ */ diff --git a/contrib/libs/sparsehash/src/sparsehash/internal/densehashtable.h b/contrib/libs/sparsehash/src/sparsehash/internal/densehashtable.h index 7faa5913b5..cdf4ff624a 100644 --- a/contrib/libs/sparsehash/src/sparsehash/internal/densehashtable.h +++ b/contrib/libs/sparsehash/src/sparsehash/internal/densehashtable.h @@ -1,1327 +1,1327 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// -// A dense hashtable is a particular implementation of -// a hashtable: one that is meant to minimize memory allocation. -// It does this by using an array to store all the data. We -// steal a value from the key space to indicate "empty" array -// elements (ie indices where no item lives) and another to indicate -// "deleted" elements. -// -// (Note it is possible to change the value of the delete key -// on the fly; you can even remove it, though after that point -// the hashtable is insert_only until you set it again. The empty -// value however can't be changed.) -// -// To minimize allocation and pointer overhead, we use internal -// probing, in which the hashtable is a single table, and collisions -// are resolved by trying to insert again in another bucket. The -// most cache-efficient internal probing schemes are linear probing -// (which suffers, alas, from clumping) and quadratic probing, which -// is what we implement by default. -// -// Type requirements: value_type is required to be Copy Constructible -// and Default Constructible. It is not required to be (and commonly -// isn't) Assignable. -// -// You probably shouldn't use this code directly. Use dense_hash_map<> -// or dense_hash_set<> instead. - -// You can change the following below: -// HT_OCCUPANCY_PCT -- how full before we double size -// HT_EMPTY_PCT -- how empty before we halve size -// HT_MIN_BUCKETS -- default smallest bucket size -// -// You can also change enlarge_factor (which defaults to -// HT_OCCUPANCY_PCT), and shrink_factor (which defaults to -// HT_EMPTY_PCT) with set_resizing_parameters(). -// -// How to decide what values to use? -// shrink_factor's default of .4 * OCCUPANCY_PCT, is probably good. -// HT_MIN_BUCKETS is probably unnecessary since you can specify -// (indirectly) the starting number of buckets at construct-time. -// For enlarge_factor, you can use this chart to try to trade-off -// expected lookup time to the space taken up. By default, this -// code uses quadratic probing, though you can change it to linear -// via JUMP_ below if you really want to. -// -// From http://www.augustana.ca/~mohrj/courses/1999.fall/csc210/lecture_notes/hashing.html -// NUMBER OF PROBES / LOOKUP Successful Unsuccessful -// Quadratic collision resolution 1 - ln(1-L) - L/2 1/(1-L) - L - ln(1-L) -// Linear collision resolution [1+1/(1-L)]/2 [1+1/(1-L)2]/2 -// -// -- enlarge_factor -- 0.10 0.50 0.60 0.75 0.80 0.90 0.99 -// QUADRATIC COLLISION RES. -// probes/successful lookup 1.05 1.44 1.62 2.01 2.21 2.85 5.11 -// probes/unsuccessful lookup 1.11 2.19 2.82 4.64 5.81 11.4 103.6 -// LINEAR COLLISION RES. -// probes/successful lookup 1.06 1.5 1.75 2.5 3.0 5.5 50.5 -// probes/unsuccessful lookup 1.12 2.5 3.6 8.5 13.0 50.0 5000.0 - -#ifndef _DENSEHASHTABLE_H_ -#define _DENSEHASHTABLE_H_ - -#include <sparsehash/internal/sparseconfig.h> -#include <assert.h> -#include <stdio.h> // for FILE, fwrite, fread -#include <algorithm> // For swap(), eg -#include <iterator> // For iterator tags -#include <limits> // for numeric_limits -#include <memory> // For uninitialized_fill -#include <utility> // for pair -#include <sparsehash/internal/hashtable-common.h> -#include <sparsehash/internal/libc_allocator_with_realloc.h> -#include <sparsehash/type_traits.h> -#include <stdexcept> // For length_error - -_START_GOOGLE_NAMESPACE_ - -namespace base { // just to make google->opensource transition easier -using GOOGLE_NAMESPACE::true_type; -using GOOGLE_NAMESPACE::false_type; -using GOOGLE_NAMESPACE::integral_constant; -using GOOGLE_NAMESPACE::is_same; -using GOOGLE_NAMESPACE::remove_const; -} - -// The probing method -// Linear probing -// #define JUMP_(key, num_probes) ( 1 ) -// Quadratic probing -#define JUMP_(key, num_probes) ( num_probes ) - -// Hashtable class, used to implement the hashed associative containers -// hash_set and hash_map. - -// Value: what is stored in the table (each bucket is a Value). -// Key: something in a 1-to-1 correspondence to a Value, that can be used -// to search for a Value in the table (find() takes a Key). -// HashFcn: Takes a Key and returns an integer, the more unique the better. -// ExtractKey: given a Value, returns the unique Key associated with it. -// Must inherit from unary_function, or at least have a -// result_type enum indicating the return type of operator(). -// SetKey: given a Value* and a Key, modifies the value such that -// ExtractKey(value) == key. We guarantee this is only called -// with key == deleted_key or key == empty_key. -// EqualKey: Given two Keys, says whether they are the same (that is, -// if they are both associated with the same Value). -// Alloc: STL allocator to use to allocate memory. - -template <class Value, class Key, class HashFcn, - class ExtractKey, class SetKey, class EqualKey, class Alloc> -class dense_hashtable; - -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -struct dense_hashtable_iterator; - -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -struct dense_hashtable_const_iterator; - -// We're just an array, but we need to skip over empty and deleted elements -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -struct dense_hashtable_iterator { - private: - typedef typename A::template rebind<V>::other value_alloc_type; - - public: - typedef dense_hashtable_iterator<V,K,HF,ExK,SetK,EqK,A> iterator; - typedef dense_hashtable_const_iterator<V,K,HF,ExK,SetK,EqK,A> const_iterator; - - typedef std::forward_iterator_tag iterator_category; // very little defined! - typedef V value_type; - typedef typename value_alloc_type::difference_type difference_type; - typedef typename value_alloc_type::size_type size_type; - typedef typename value_alloc_type::reference reference; - typedef typename value_alloc_type::pointer pointer; - - // "Real" constructor and default constructor - dense_hashtable_iterator(const dense_hashtable<V,K,HF,ExK,SetK,EqK,A> *h, - pointer it, pointer it_end, bool advance) - : ht(h), pos(it), end(it_end) { - if (advance) advance_past_empty_and_deleted(); - } - dense_hashtable_iterator() { } - // The default destructor is fine; we don't define one - // The default operator= is fine; we don't define one - - // Happy dereferencer - reference operator*() const { return *pos; } - pointer operator->() const { return &(operator*()); } - - // Arithmetic. The only hard part is making sure that - // we're not on an empty or marked-deleted array element - void advance_past_empty_and_deleted() { - while ( pos != end && (ht->test_empty(*this) || ht->test_deleted(*this)) ) - ++pos; - } - iterator& operator++() { - assert(pos != end); ++pos; advance_past_empty_and_deleted(); return *this; - } - iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } - - // Comparison. - bool operator==(const iterator& it) const { return pos == it.pos; } - bool operator!=(const iterator& it) const { return pos != it.pos; } - - - // The actual data - const dense_hashtable<V,K,HF,ExK,SetK,EqK,A> *ht; - pointer pos, end; -}; - - -// Now do it all again, but with const-ness! -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -struct dense_hashtable_const_iterator { - private: - typedef typename A::template rebind<V>::other value_alloc_type; - - public: - typedef dense_hashtable_iterator<V,K,HF,ExK,SetK,EqK,A> iterator; - typedef dense_hashtable_const_iterator<V,K,HF,ExK,SetK,EqK,A> const_iterator; - - typedef std::forward_iterator_tag iterator_category; // very little defined! - typedef V value_type; - typedef typename value_alloc_type::difference_type difference_type; - typedef typename value_alloc_type::size_type size_type; - typedef typename value_alloc_type::const_reference reference; - typedef typename value_alloc_type::const_pointer pointer; - - // "Real" constructor and default constructor - dense_hashtable_const_iterator( - const dense_hashtable<V,K,HF,ExK,SetK,EqK,A> *h, - pointer it, pointer it_end, bool advance) - : ht(h), pos(it), end(it_end) { - if (advance) advance_past_empty_and_deleted(); - } - dense_hashtable_const_iterator() - : ht(NULL), pos(pointer()), end(pointer()) { } - // This lets us convert regular iterators to const iterators - dense_hashtable_const_iterator(const iterator &it) - : ht(it.ht), pos(it.pos), end(it.end) { } - // The default destructor is fine; we don't define one - // The default operator= is fine; we don't define one - - // Happy dereferencer - reference operator*() const { return *pos; } - pointer operator->() const { return &(operator*()); } - - // Arithmetic. The only hard part is making sure that - // we're not on an empty or marked-deleted array element - void advance_past_empty_and_deleted() { - while ( pos != end && (ht->test_empty(*this) || ht->test_deleted(*this)) ) - ++pos; - } - const_iterator& operator++() { - assert(pos != end); ++pos; advance_past_empty_and_deleted(); return *this; - } - const_iterator operator++(int) { const_iterator tmp(*this); ++*this; return tmp; } - - // Comparison. - bool operator==(const const_iterator& it) const { return pos == it.pos; } - bool operator!=(const const_iterator& it) const { return pos != it.pos; } - - - // The actual data - const dense_hashtable<V,K,HF,ExK,SetK,EqK,A> *ht; - pointer pos, end; -}; - -template <class Value, class Key, class HashFcn, - class ExtractKey, class SetKey, class EqualKey, class Alloc> -class dense_hashtable { - private: - typedef typename Alloc::template rebind<Value>::other value_alloc_type; - - public: - typedef Key key_type; - typedef Value value_type; - typedef HashFcn hasher; - typedef EqualKey key_equal; - typedef Alloc allocator_type; - - typedef typename value_alloc_type::size_type size_type; - typedef typename value_alloc_type::difference_type difference_type; - typedef typename value_alloc_type::reference reference; - typedef typename value_alloc_type::const_reference const_reference; - typedef typename value_alloc_type::pointer pointer; - typedef typename value_alloc_type::const_pointer const_pointer; - typedef dense_hashtable_iterator<Value, Key, HashFcn, - ExtractKey, SetKey, EqualKey, Alloc> - iterator; - - typedef dense_hashtable_const_iterator<Value, Key, HashFcn, - ExtractKey, SetKey, EqualKey, Alloc> - const_iterator; - - // These come from tr1. For us they're the same as regular iterators. - typedef iterator local_iterator; - typedef const_iterator const_local_iterator; - - // How full we let the table get before we resize, by default. - // Knuth says .8 is good -- higher causes us to probe too much, - // though it saves memory. - static const int HT_OCCUPANCY_PCT; // defined at the bottom of this file - - // How empty we let the table get before we resize lower, by default. - // (0.0 means never resize lower.) - // It should be less than OCCUPANCY_PCT / 2 or we thrash resizing - static const int HT_EMPTY_PCT; // defined at the bottom of this file - - // Minimum size we're willing to let hashtables be. - // Must be a power of two, and at least 4. - // Note, however, that for a given hashtable, the initial size is a - // function of the first constructor arg, and may be >HT_MIN_BUCKETS. - static const size_type HT_MIN_BUCKETS = 4; - - // By default, if you don't specify a hashtable size at - // construction-time, we use this size. Must be a power of two, and - // at least HT_MIN_BUCKETS. - static const size_type HT_DEFAULT_STARTING_BUCKETS = 32; - - // ITERATOR FUNCTIONS - iterator begin() { return iterator(this, table, - table + num_buckets, true); } - iterator end() { return iterator(this, table + num_buckets, - table + num_buckets, true); } - const_iterator begin() const { return const_iterator(this, table, - table+num_buckets,true);} - const_iterator end() const { return const_iterator(this, table + num_buckets, - table+num_buckets,true);} - - // These come from tr1 unordered_map. They iterate over 'bucket' n. - // We'll just consider bucket n to be the n-th element of the table. - local_iterator begin(size_type i) { - return local_iterator(this, table + i, table + i+1, false); - } - local_iterator end(size_type i) { - local_iterator it = begin(i); - if (!test_empty(i) && !test_deleted(i)) - ++it; - return it; - } - const_local_iterator begin(size_type i) const { - return const_local_iterator(this, table + i, table + i+1, false); - } - const_local_iterator end(size_type i) const { - const_local_iterator it = begin(i); - if (!test_empty(i) && !test_deleted(i)) - ++it; - return it; - } - - // ACCESSOR FUNCTIONS for the things we templatize on, basically - hasher hash_funct() const { return settings; } - key_equal key_eq() const { return key_info; } - allocator_type get_allocator() const { - return allocator_type(val_info); - } - - // Accessor function for statistics gathering. - int num_table_copies() const { return settings.num_ht_copies(); } - - private: - // Annoyingly, we can't copy values around, because they might have - // const components (they're probably pair<const X, Y>). We use - // explicit destructor invocation and placement new to get around - // this. Arg. - void set_value(pointer dst, const_reference src) { - dst->~value_type(); // delete the old value, if any - new(dst) value_type(src); - } - - void destroy_buckets(size_type first, size_type last) { - for ( ; first != last; ++first) - table[first].~value_type(); - } - - // DELETE HELPER FUNCTIONS - // This lets the user describe a key that will indicate deleted - // table entries. This key should be an "impossible" entry -- - // if you try to insert it for real, you won't be able to retrieve it! - // (NB: while you pass in an entire value, only the key part is looked - // at. This is just because I don't know how to assign just a key.) - private: - void squash_deleted() { // gets rid of any deleted entries we have - if ( num_deleted ) { // get rid of deleted before writing - dense_hashtable tmp(*this); // copying will get rid of deleted - swap(tmp); // now we are tmp - } - assert(num_deleted == 0); - } - - // Test if the given key is the deleted indicator. Requires - // num_deleted > 0, for correctness of read(), and because that - // guarantees that key_info.delkey is valid. - bool test_deleted_key(const key_type& key) const { - assert(num_deleted > 0); - return equals(key_info.delkey, key); - } - - public: - void set_deleted_key(const key_type &key) { - // the empty indicator (if specified) and the deleted indicator - // must be different - assert((!settings.use_empty() || !equals(key, get_key(val_info.emptyval))) - && "Passed the empty-key to set_deleted_key"); - // It's only safe to change what "deleted" means if we purge deleted guys - squash_deleted(); - settings.set_use_deleted(true); - key_info.delkey = key; - } - void clear_deleted_key() { - squash_deleted(); - settings.set_use_deleted(false); - } - key_type deleted_key() const { - assert(settings.use_deleted() - && "Must set deleted key before calling deleted_key"); - return key_info.delkey; - } - - // These are public so the iterators can use them - // True if the item at position bucknum is "deleted" marker - bool test_deleted(size_type bucknum) const { - // Invariant: !use_deleted() implies num_deleted is 0. - assert(settings.use_deleted() || num_deleted == 0); - return num_deleted > 0 && test_deleted_key(get_key(table[bucknum])); - } - bool test_deleted(const iterator &it) const { - // Invariant: !use_deleted() implies num_deleted is 0. - assert(settings.use_deleted() || num_deleted == 0); - return num_deleted > 0 && test_deleted_key(get_key(*it)); - } - bool test_deleted(const const_iterator &it) const { - // Invariant: !use_deleted() implies num_deleted is 0. - assert(settings.use_deleted() || num_deleted == 0); - return num_deleted > 0 && test_deleted_key(get_key(*it)); - } - - private: - void check_use_deleted(const char* caller) { - (void)caller; // could log it if the assert failed - assert(settings.use_deleted()); - } - - // Set it so test_deleted is true. true if object didn't used to be deleted. - bool set_deleted(iterator &it) { - check_use_deleted("set_deleted()"); - bool retval = !test_deleted(it); - // &* converts from iterator to value-type. - set_key(&(*it), key_info.delkey); - return retval; - } - // Set it so test_deleted is false. true if object used to be deleted. - bool clear_deleted(iterator &it) { - check_use_deleted("clear_deleted()"); - // Happens automatically when we assign something else in its place. - return test_deleted(it); - } - - // We also allow to set/clear the deleted bit on a const iterator. - // We allow a const_iterator for the same reason you can delete a - // const pointer: it's convenient, and semantically you can't use - // 'it' after it's been deleted anyway, so its const-ness doesn't - // really matter. - bool set_deleted(const_iterator &it) { - check_use_deleted("set_deleted()"); - bool retval = !test_deleted(it); - set_key(const_cast<pointer>(&(*it)), key_info.delkey); - return retval; - } - // Set it so test_deleted is false. true if object used to be deleted. - bool clear_deleted(const_iterator &it) { - check_use_deleted("clear_deleted()"); - return test_deleted(it); - } - - // EMPTY HELPER FUNCTIONS - // This lets the user describe a key that will indicate empty (unused) - // table entries. This key should be an "impossible" entry -- - // if you try to insert it for real, you won't be able to retrieve it! - // (NB: while you pass in an entire value, only the key part is looked - // at. This is just because I don't know how to assign just a key.) - public: - // These are public so the iterators can use them - // True if the item at position bucknum is "empty" marker - bool test_empty(size_type bucknum) const { - assert(settings.use_empty()); // we always need to know what's empty! - return equals(get_key(val_info.emptyval), get_key(table[bucknum])); - } - bool test_empty(const iterator &it) const { - assert(settings.use_empty()); // we always need to know what's empty! - return equals(get_key(val_info.emptyval), get_key(*it)); - } - bool test_empty(const const_iterator &it) const { - assert(settings.use_empty()); // we always need to know what's empty! - return equals(get_key(val_info.emptyval), get_key(*it)); - } - - private: - void fill_range_with_empty(pointer table_start, pointer table_end) { - std::uninitialized_fill(table_start, table_end, val_info.emptyval); - } - - public: - // TODO(csilvers): change all callers of this to pass in a key instead, - // and take a const key_type instead of const value_type. - void set_empty_key(const_reference val) { - // Once you set the empty key, you can't change it - assert(!settings.use_empty() && "Calling set_empty_key multiple times"); - // The deleted indicator (if specified) and the empty indicator - // must be different. - assert((!settings.use_deleted() || !equals(get_key(val), key_info.delkey)) - && "Setting the empty key the same as the deleted key"); - settings.set_use_empty(true); - set_value(&val_info.emptyval, val); - - assert(!table); // must set before first use - // num_buckets was set in constructor even though table was NULL - table = val_info.allocate(num_buckets); - assert(table); - fill_range_with_empty(table, table + num_buckets); - } - // TODO(user): return a key_type rather than a value_type - value_type empty_key() const { - assert(settings.use_empty()); - return val_info.emptyval; - } - - // FUNCTIONS CONCERNING SIZE - public: - size_type size() const { return num_elements - num_deleted; } - size_type max_size() const { return val_info.max_size(); } - bool empty() const { return size() == 0; } - size_type bucket_count() const { return num_buckets; } - size_type max_bucket_count() const { return max_size(); } - size_type nonempty_bucket_count() const { return num_elements; } - // These are tr1 methods. Their idea of 'bucket' doesn't map well to - // what we do. We just say every bucket has 0 or 1 items in it. - size_type bucket_size(size_type i) const { - return begin(i) == end(i) ? 0 : 1; - } - - private: - // Because of the above, size_type(-1) is never legal; use it for errors - static const size_type ILLEGAL_BUCKET = size_type(-1); - - // Used after a string of deletes. Returns true if we actually shrunk. - // TODO(csilvers): take a delta so we can take into account inserts - // done after shrinking. Maybe make part of the Settings class? - bool maybe_shrink() { - assert(num_elements >= num_deleted); - assert((bucket_count() & (bucket_count()-1)) == 0); // is a power of two - assert(bucket_count() >= HT_MIN_BUCKETS); - bool retval = false; - - // If you construct a hashtable with < HT_DEFAULT_STARTING_BUCKETS, - // we'll never shrink until you get relatively big, and we'll never - // shrink below HT_DEFAULT_STARTING_BUCKETS. Otherwise, something - // like "dense_hash_set<int> x; x.insert(4); x.erase(4);" will - // shrink us down to HT_MIN_BUCKETS buckets, which is too small. - const size_type num_remain = num_elements - num_deleted; - const size_type shrink_threshold = settings.shrink_threshold(); - if (shrink_threshold > 0 && num_remain < shrink_threshold && - bucket_count() > HT_DEFAULT_STARTING_BUCKETS) { - const float shrink_factor = settings.shrink_factor(); - size_type sz = bucket_count() / 2; // find how much we should shrink - while (sz > HT_DEFAULT_STARTING_BUCKETS && - num_remain < sz * shrink_factor) { - sz /= 2; // stay a power of 2 - } - dense_hashtable tmp(*this, sz); // Do the actual resizing - swap(tmp); // now we are tmp - retval = true; - } - settings.set_consider_shrink(false); // because we just considered it - return retval; - } - - // We'll let you resize a hashtable -- though this makes us copy all! - // When you resize, you say, "make it big enough for this many more elements" - // Returns true if we actually resized, false if size was already ok. - bool resize_delta(size_type delta) { - bool did_resize = false; - if ( settings.consider_shrink() ) { // see if lots of deletes happened - if ( maybe_shrink() ) - did_resize = true; - } - if (num_elements >= - (std::numeric_limits<size_type>::max)() - delta) { - throw std::length_error("resize overflow"); - } - if ( bucket_count() >= HT_MIN_BUCKETS && - (num_elements + delta) <= settings.enlarge_threshold() ) - return did_resize; // we're ok as we are - - // Sometimes, we need to resize just to get rid of all the - // "deleted" buckets that are clogging up the hashtable. So when - // deciding whether to resize, count the deleted buckets (which - // are currently taking up room). But later, when we decide what - // size to resize to, *don't* count deleted buckets, since they - // get discarded during the resize. +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// +// A dense hashtable is a particular implementation of +// a hashtable: one that is meant to minimize memory allocation. +// It does this by using an array to store all the data. We +// steal a value from the key space to indicate "empty" array +// elements (ie indices where no item lives) and another to indicate +// "deleted" elements. +// +// (Note it is possible to change the value of the delete key +// on the fly; you can even remove it, though after that point +// the hashtable is insert_only until you set it again. The empty +// value however can't be changed.) +// +// To minimize allocation and pointer overhead, we use internal +// probing, in which the hashtable is a single table, and collisions +// are resolved by trying to insert again in another bucket. The +// most cache-efficient internal probing schemes are linear probing +// (which suffers, alas, from clumping) and quadratic probing, which +// is what we implement by default. +// +// Type requirements: value_type is required to be Copy Constructible +// and Default Constructible. It is not required to be (and commonly +// isn't) Assignable. +// +// You probably shouldn't use this code directly. Use dense_hash_map<> +// or dense_hash_set<> instead. + +// You can change the following below: +// HT_OCCUPANCY_PCT -- how full before we double size +// HT_EMPTY_PCT -- how empty before we halve size +// HT_MIN_BUCKETS -- default smallest bucket size +// +// You can also change enlarge_factor (which defaults to +// HT_OCCUPANCY_PCT), and shrink_factor (which defaults to +// HT_EMPTY_PCT) with set_resizing_parameters(). +// +// How to decide what values to use? +// shrink_factor's default of .4 * OCCUPANCY_PCT, is probably good. +// HT_MIN_BUCKETS is probably unnecessary since you can specify +// (indirectly) the starting number of buckets at construct-time. +// For enlarge_factor, you can use this chart to try to trade-off +// expected lookup time to the space taken up. By default, this +// code uses quadratic probing, though you can change it to linear +// via JUMP_ below if you really want to. +// +// From http://www.augustana.ca/~mohrj/courses/1999.fall/csc210/lecture_notes/hashing.html +// NUMBER OF PROBES / LOOKUP Successful Unsuccessful +// Quadratic collision resolution 1 - ln(1-L) - L/2 1/(1-L) - L - ln(1-L) +// Linear collision resolution [1+1/(1-L)]/2 [1+1/(1-L)2]/2 +// +// -- enlarge_factor -- 0.10 0.50 0.60 0.75 0.80 0.90 0.99 +// QUADRATIC COLLISION RES. +// probes/successful lookup 1.05 1.44 1.62 2.01 2.21 2.85 5.11 +// probes/unsuccessful lookup 1.11 2.19 2.82 4.64 5.81 11.4 103.6 +// LINEAR COLLISION RES. +// probes/successful lookup 1.06 1.5 1.75 2.5 3.0 5.5 50.5 +// probes/unsuccessful lookup 1.12 2.5 3.6 8.5 13.0 50.0 5000.0 + +#ifndef _DENSEHASHTABLE_H_ +#define _DENSEHASHTABLE_H_ + +#include <sparsehash/internal/sparseconfig.h> +#include <assert.h> +#include <stdio.h> // for FILE, fwrite, fread +#include <algorithm> // For swap(), eg +#include <iterator> // For iterator tags +#include <limits> // for numeric_limits +#include <memory> // For uninitialized_fill +#include <utility> // for pair +#include <sparsehash/internal/hashtable-common.h> +#include <sparsehash/internal/libc_allocator_with_realloc.h> +#include <sparsehash/type_traits.h> +#include <stdexcept> // For length_error + +_START_GOOGLE_NAMESPACE_ + +namespace base { // just to make google->opensource transition easier +using GOOGLE_NAMESPACE::true_type; +using GOOGLE_NAMESPACE::false_type; +using GOOGLE_NAMESPACE::integral_constant; +using GOOGLE_NAMESPACE::is_same; +using GOOGLE_NAMESPACE::remove_const; +} + +// The probing method +// Linear probing +// #define JUMP_(key, num_probes) ( 1 ) +// Quadratic probing +#define JUMP_(key, num_probes) ( num_probes ) + +// Hashtable class, used to implement the hashed associative containers +// hash_set and hash_map. + +// Value: what is stored in the table (each bucket is a Value). +// Key: something in a 1-to-1 correspondence to a Value, that can be used +// to search for a Value in the table (find() takes a Key). +// HashFcn: Takes a Key and returns an integer, the more unique the better. +// ExtractKey: given a Value, returns the unique Key associated with it. +// Must inherit from unary_function, or at least have a +// result_type enum indicating the return type of operator(). +// SetKey: given a Value* and a Key, modifies the value such that +// ExtractKey(value) == key. We guarantee this is only called +// with key == deleted_key or key == empty_key. +// EqualKey: Given two Keys, says whether they are the same (that is, +// if they are both associated with the same Value). +// Alloc: STL allocator to use to allocate memory. + +template <class Value, class Key, class HashFcn, + class ExtractKey, class SetKey, class EqualKey, class Alloc> +class dense_hashtable; + +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +struct dense_hashtable_iterator; + +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +struct dense_hashtable_const_iterator; + +// We're just an array, but we need to skip over empty and deleted elements +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +struct dense_hashtable_iterator { + private: + typedef typename A::template rebind<V>::other value_alloc_type; + + public: + typedef dense_hashtable_iterator<V,K,HF,ExK,SetK,EqK,A> iterator; + typedef dense_hashtable_const_iterator<V,K,HF,ExK,SetK,EqK,A> const_iterator; + + typedef std::forward_iterator_tag iterator_category; // very little defined! + typedef V value_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::pointer pointer; + + // "Real" constructor and default constructor + dense_hashtable_iterator(const dense_hashtable<V,K,HF,ExK,SetK,EqK,A> *h, + pointer it, pointer it_end, bool advance) + : ht(h), pos(it), end(it_end) { + if (advance) advance_past_empty_and_deleted(); + } + dense_hashtable_iterator() { } + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *pos; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic. The only hard part is making sure that + // we're not on an empty or marked-deleted array element + void advance_past_empty_and_deleted() { + while ( pos != end && (ht->test_empty(*this) || ht->test_deleted(*this)) ) + ++pos; + } + iterator& operator++() { + assert(pos != end); ++pos; advance_past_empty_and_deleted(); return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + + // Comparison. + bool operator==(const iterator& it) const { return pos == it.pos; } + bool operator!=(const iterator& it) const { return pos != it.pos; } + + + // The actual data + const dense_hashtable<V,K,HF,ExK,SetK,EqK,A> *ht; + pointer pos, end; +}; + + +// Now do it all again, but with const-ness! +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +struct dense_hashtable_const_iterator { + private: + typedef typename A::template rebind<V>::other value_alloc_type; + + public: + typedef dense_hashtable_iterator<V,K,HF,ExK,SetK,EqK,A> iterator; + typedef dense_hashtable_const_iterator<V,K,HF,ExK,SetK,EqK,A> const_iterator; + + typedef std::forward_iterator_tag iterator_category; // very little defined! + typedef V value_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::const_reference reference; + typedef typename value_alloc_type::const_pointer pointer; + + // "Real" constructor and default constructor + dense_hashtable_const_iterator( + const dense_hashtable<V,K,HF,ExK,SetK,EqK,A> *h, + pointer it, pointer it_end, bool advance) + : ht(h), pos(it), end(it_end) { + if (advance) advance_past_empty_and_deleted(); + } + dense_hashtable_const_iterator() + : ht(NULL), pos(pointer()), end(pointer()) { } + // This lets us convert regular iterators to const iterators + dense_hashtable_const_iterator(const iterator &it) + : ht(it.ht), pos(it.pos), end(it.end) { } + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *pos; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic. The only hard part is making sure that + // we're not on an empty or marked-deleted array element + void advance_past_empty_and_deleted() { + while ( pos != end && (ht->test_empty(*this) || ht->test_deleted(*this)) ) + ++pos; + } + const_iterator& operator++() { + assert(pos != end); ++pos; advance_past_empty_and_deleted(); return *this; + } + const_iterator operator++(int) { const_iterator tmp(*this); ++*this; return tmp; } + + // Comparison. + bool operator==(const const_iterator& it) const { return pos == it.pos; } + bool operator!=(const const_iterator& it) const { return pos != it.pos; } + + + // The actual data + const dense_hashtable<V,K,HF,ExK,SetK,EqK,A> *ht; + pointer pos, end; +}; + +template <class Value, class Key, class HashFcn, + class ExtractKey, class SetKey, class EqualKey, class Alloc> +class dense_hashtable { + private: + typedef typename Alloc::template rebind<Value>::other value_alloc_type; + + public: + typedef Key key_type; + typedef Value value_type; + typedef HashFcn hasher; + typedef EqualKey key_equal; + typedef Alloc allocator_type; + + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::const_reference const_reference; + typedef typename value_alloc_type::pointer pointer; + typedef typename value_alloc_type::const_pointer const_pointer; + typedef dense_hashtable_iterator<Value, Key, HashFcn, + ExtractKey, SetKey, EqualKey, Alloc> + iterator; + + typedef dense_hashtable_const_iterator<Value, Key, HashFcn, + ExtractKey, SetKey, EqualKey, Alloc> + const_iterator; + + // These come from tr1. For us they're the same as regular iterators. + typedef iterator local_iterator; + typedef const_iterator const_local_iterator; + + // How full we let the table get before we resize, by default. + // Knuth says .8 is good -- higher causes us to probe too much, + // though it saves memory. + static const int HT_OCCUPANCY_PCT; // defined at the bottom of this file + + // How empty we let the table get before we resize lower, by default. + // (0.0 means never resize lower.) + // It should be less than OCCUPANCY_PCT / 2 or we thrash resizing + static const int HT_EMPTY_PCT; // defined at the bottom of this file + + // Minimum size we're willing to let hashtables be. + // Must be a power of two, and at least 4. + // Note, however, that for a given hashtable, the initial size is a + // function of the first constructor arg, and may be >HT_MIN_BUCKETS. + static const size_type HT_MIN_BUCKETS = 4; + + // By default, if you don't specify a hashtable size at + // construction-time, we use this size. Must be a power of two, and + // at least HT_MIN_BUCKETS. + static const size_type HT_DEFAULT_STARTING_BUCKETS = 32; + + // ITERATOR FUNCTIONS + iterator begin() { return iterator(this, table, + table + num_buckets, true); } + iterator end() { return iterator(this, table + num_buckets, + table + num_buckets, true); } + const_iterator begin() const { return const_iterator(this, table, + table+num_buckets,true);} + const_iterator end() const { return const_iterator(this, table + num_buckets, + table+num_buckets,true);} + + // These come from tr1 unordered_map. They iterate over 'bucket' n. + // We'll just consider bucket n to be the n-th element of the table. + local_iterator begin(size_type i) { + return local_iterator(this, table + i, table + i+1, false); + } + local_iterator end(size_type i) { + local_iterator it = begin(i); + if (!test_empty(i) && !test_deleted(i)) + ++it; + return it; + } + const_local_iterator begin(size_type i) const { + return const_local_iterator(this, table + i, table + i+1, false); + } + const_local_iterator end(size_type i) const { + const_local_iterator it = begin(i); + if (!test_empty(i) && !test_deleted(i)) + ++it; + return it; + } + + // ACCESSOR FUNCTIONS for the things we templatize on, basically + hasher hash_funct() const { return settings; } + key_equal key_eq() const { return key_info; } + allocator_type get_allocator() const { + return allocator_type(val_info); + } + + // Accessor function for statistics gathering. + int num_table_copies() const { return settings.num_ht_copies(); } + + private: + // Annoyingly, we can't copy values around, because they might have + // const components (they're probably pair<const X, Y>). We use + // explicit destructor invocation and placement new to get around + // this. Arg. + void set_value(pointer dst, const_reference src) { + dst->~value_type(); // delete the old value, if any + new(dst) value_type(src); + } + + void destroy_buckets(size_type first, size_type last) { + for ( ; first != last; ++first) + table[first].~value_type(); + } + + // DELETE HELPER FUNCTIONS + // This lets the user describe a key that will indicate deleted + // table entries. This key should be an "impossible" entry -- + // if you try to insert it for real, you won't be able to retrieve it! + // (NB: while you pass in an entire value, only the key part is looked + // at. This is just because I don't know how to assign just a key.) + private: + void squash_deleted() { // gets rid of any deleted entries we have + if ( num_deleted ) { // get rid of deleted before writing + dense_hashtable tmp(*this); // copying will get rid of deleted + swap(tmp); // now we are tmp + } + assert(num_deleted == 0); + } + + // Test if the given key is the deleted indicator. Requires + // num_deleted > 0, for correctness of read(), and because that + // guarantees that key_info.delkey is valid. + bool test_deleted_key(const key_type& key) const { + assert(num_deleted > 0); + return equals(key_info.delkey, key); + } + + public: + void set_deleted_key(const key_type &key) { + // the empty indicator (if specified) and the deleted indicator + // must be different + assert((!settings.use_empty() || !equals(key, get_key(val_info.emptyval))) + && "Passed the empty-key to set_deleted_key"); + // It's only safe to change what "deleted" means if we purge deleted guys + squash_deleted(); + settings.set_use_deleted(true); + key_info.delkey = key; + } + void clear_deleted_key() { + squash_deleted(); + settings.set_use_deleted(false); + } + key_type deleted_key() const { + assert(settings.use_deleted() + && "Must set deleted key before calling deleted_key"); + return key_info.delkey; + } + + // These are public so the iterators can use them + // True if the item at position bucknum is "deleted" marker + bool test_deleted(size_type bucknum) const { + // Invariant: !use_deleted() implies num_deleted is 0. + assert(settings.use_deleted() || num_deleted == 0); + return num_deleted > 0 && test_deleted_key(get_key(table[bucknum])); + } + bool test_deleted(const iterator &it) const { + // Invariant: !use_deleted() implies num_deleted is 0. + assert(settings.use_deleted() || num_deleted == 0); + return num_deleted > 0 && test_deleted_key(get_key(*it)); + } + bool test_deleted(const const_iterator &it) const { + // Invariant: !use_deleted() implies num_deleted is 0. + assert(settings.use_deleted() || num_deleted == 0); + return num_deleted > 0 && test_deleted_key(get_key(*it)); + } + + private: + void check_use_deleted(const char* caller) { + (void)caller; // could log it if the assert failed + assert(settings.use_deleted()); + } + + // Set it so test_deleted is true. true if object didn't used to be deleted. + bool set_deleted(iterator &it) { + check_use_deleted("set_deleted()"); + bool retval = !test_deleted(it); + // &* converts from iterator to value-type. + set_key(&(*it), key_info.delkey); + return retval; + } + // Set it so test_deleted is false. true if object used to be deleted. + bool clear_deleted(iterator &it) { + check_use_deleted("clear_deleted()"); + // Happens automatically when we assign something else in its place. + return test_deleted(it); + } + + // We also allow to set/clear the deleted bit on a const iterator. + // We allow a const_iterator for the same reason you can delete a + // const pointer: it's convenient, and semantically you can't use + // 'it' after it's been deleted anyway, so its const-ness doesn't + // really matter. + bool set_deleted(const_iterator &it) { + check_use_deleted("set_deleted()"); + bool retval = !test_deleted(it); + set_key(const_cast<pointer>(&(*it)), key_info.delkey); + return retval; + } + // Set it so test_deleted is false. true if object used to be deleted. + bool clear_deleted(const_iterator &it) { + check_use_deleted("clear_deleted()"); + return test_deleted(it); + } + + // EMPTY HELPER FUNCTIONS + // This lets the user describe a key that will indicate empty (unused) + // table entries. This key should be an "impossible" entry -- + // if you try to insert it for real, you won't be able to retrieve it! + // (NB: while you pass in an entire value, only the key part is looked + // at. This is just because I don't know how to assign just a key.) + public: + // These are public so the iterators can use them + // True if the item at position bucknum is "empty" marker + bool test_empty(size_type bucknum) const { + assert(settings.use_empty()); // we always need to know what's empty! + return equals(get_key(val_info.emptyval), get_key(table[bucknum])); + } + bool test_empty(const iterator &it) const { + assert(settings.use_empty()); // we always need to know what's empty! + return equals(get_key(val_info.emptyval), get_key(*it)); + } + bool test_empty(const const_iterator &it) const { + assert(settings.use_empty()); // we always need to know what's empty! + return equals(get_key(val_info.emptyval), get_key(*it)); + } + + private: + void fill_range_with_empty(pointer table_start, pointer table_end) { + std::uninitialized_fill(table_start, table_end, val_info.emptyval); + } + + public: + // TODO(csilvers): change all callers of this to pass in a key instead, + // and take a const key_type instead of const value_type. + void set_empty_key(const_reference val) { + // Once you set the empty key, you can't change it + assert(!settings.use_empty() && "Calling set_empty_key multiple times"); + // The deleted indicator (if specified) and the empty indicator + // must be different. + assert((!settings.use_deleted() || !equals(get_key(val), key_info.delkey)) + && "Setting the empty key the same as the deleted key"); + settings.set_use_empty(true); + set_value(&val_info.emptyval, val); + + assert(!table); // must set before first use + // num_buckets was set in constructor even though table was NULL + table = val_info.allocate(num_buckets); + assert(table); + fill_range_with_empty(table, table + num_buckets); + } + // TODO(user): return a key_type rather than a value_type + value_type empty_key() const { + assert(settings.use_empty()); + return val_info.emptyval; + } + + // FUNCTIONS CONCERNING SIZE + public: + size_type size() const { return num_elements - num_deleted; } + size_type max_size() const { return val_info.max_size(); } + bool empty() const { return size() == 0; } + size_type bucket_count() const { return num_buckets; } + size_type max_bucket_count() const { return max_size(); } + size_type nonempty_bucket_count() const { return num_elements; } + // These are tr1 methods. Their idea of 'bucket' doesn't map well to + // what we do. We just say every bucket has 0 or 1 items in it. + size_type bucket_size(size_type i) const { + return begin(i) == end(i) ? 0 : 1; + } + + private: + // Because of the above, size_type(-1) is never legal; use it for errors + static const size_type ILLEGAL_BUCKET = size_type(-1); + + // Used after a string of deletes. Returns true if we actually shrunk. + // TODO(csilvers): take a delta so we can take into account inserts + // done after shrinking. Maybe make part of the Settings class? + bool maybe_shrink() { + assert(num_elements >= num_deleted); + assert((bucket_count() & (bucket_count()-1)) == 0); // is a power of two + assert(bucket_count() >= HT_MIN_BUCKETS); + bool retval = false; + + // If you construct a hashtable with < HT_DEFAULT_STARTING_BUCKETS, + // we'll never shrink until you get relatively big, and we'll never + // shrink below HT_DEFAULT_STARTING_BUCKETS. Otherwise, something + // like "dense_hash_set<int> x; x.insert(4); x.erase(4);" will + // shrink us down to HT_MIN_BUCKETS buckets, which is too small. + const size_type num_remain = num_elements - num_deleted; + const size_type shrink_threshold = settings.shrink_threshold(); + if (shrink_threshold > 0 && num_remain < shrink_threshold && + bucket_count() > HT_DEFAULT_STARTING_BUCKETS) { + const float shrink_factor = settings.shrink_factor(); + size_type sz = bucket_count() / 2; // find how much we should shrink + while (sz > HT_DEFAULT_STARTING_BUCKETS && + num_remain < sz * shrink_factor) { + sz /= 2; // stay a power of 2 + } + dense_hashtable tmp(*this, sz); // Do the actual resizing + swap(tmp); // now we are tmp + retval = true; + } + settings.set_consider_shrink(false); // because we just considered it + return retval; + } + + // We'll let you resize a hashtable -- though this makes us copy all! + // When you resize, you say, "make it big enough for this many more elements" + // Returns true if we actually resized, false if size was already ok. + bool resize_delta(size_type delta) { + bool did_resize = false; + if ( settings.consider_shrink() ) { // see if lots of deletes happened + if ( maybe_shrink() ) + did_resize = true; + } + if (num_elements >= + (std::numeric_limits<size_type>::max)() - delta) { + throw std::length_error("resize overflow"); + } + if ( bucket_count() >= HT_MIN_BUCKETS && + (num_elements + delta) <= settings.enlarge_threshold() ) + return did_resize; // we're ok as we are + + // Sometimes, we need to resize just to get rid of all the + // "deleted" buckets that are clogging up the hashtable. So when + // deciding whether to resize, count the deleted buckets (which + // are currently taking up room). But later, when we decide what + // size to resize to, *don't* count deleted buckets, since they + // get discarded during the resize. size_type needed_size = settings.min_buckets(num_elements + delta, 0); - if ( needed_size <= bucket_count() ) // we have enough buckets - return did_resize; - - size_type resize_to = - settings.min_buckets(num_elements - num_deleted + delta, bucket_count()); - + if ( needed_size <= bucket_count() ) // we have enough buckets + return did_resize; + + size_type resize_to = + settings.min_buckets(num_elements - num_deleted + delta, bucket_count()); + // When num_deleted is large, we may still grow but we do not want to // over expand. So we reduce needed_size by a portion of num_deleted // (the exact portion does not matter). This is especially helpful // when min_load_factor is zero (no shrink at all) to avoid doubling // the bucket count to infinity. See also test ResizeWithoutShrink. needed_size = settings.min_buckets(num_elements - num_deleted / 4 + delta, 0); - if (resize_to < needed_size && // may double resize_to - resize_to < (std::numeric_limits<size_type>::max)() / 2) { - // This situation means that we have enough deleted elements, - // that once we purge them, we won't actually have needed to - // grow. But we may want to grow anyway: if we just purge one - // element, say, we'll have to grow anyway next time we - // insert. Might as well grow now, since we're already going - // through the trouble of copying (in order to purge the - // deleted elements). - const size_type target = - static_cast<size_type>(settings.shrink_size(resize_to*2)); - if (num_elements - num_deleted + delta >= target) { - // Good, we won't be below the shrink threshhold even if we double. - resize_to *= 2; - } - } - dense_hashtable tmp(*this, resize_to); - swap(tmp); // now we are tmp - return true; - } - - // We require table be not-NULL and empty before calling this. - void resize_table(size_type /*old_size*/, size_type new_size, - base::true_type) { - table = val_info.realloc_or_die(table, new_size); - } - - void resize_table(size_type old_size, size_type new_size, base::false_type) { - val_info.deallocate(table, old_size); - table = val_info.allocate(new_size); - } - - // Used to actually do the rehashing when we grow/shrink a hashtable - void copy_from(const dense_hashtable &ht, size_type min_buckets_wanted) { - clear_to_size(settings.min_buckets(ht.size(), min_buckets_wanted)); - - // We use a normal iterator to get non-deleted bcks from ht - // We could use insert() here, but since we know there are - // no duplicates and no deleted items, we can be more efficient - assert((bucket_count() & (bucket_count()-1)) == 0); // a power of two - for ( const_iterator it = ht.begin(); it != ht.end(); ++it ) { - size_type num_probes = 0; // how many times we've probed - size_type bucknum; - const size_type bucket_count_minus_one = bucket_count() - 1; - for (bucknum = hash(get_key(*it)) & bucket_count_minus_one; - !test_empty(bucknum); // not empty - bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one) { - ++num_probes; - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - set_value(&table[bucknum], *it); // copies the value to here - num_elements++; - } - settings.inc_num_ht_copies(); - } - - // Required by the spec for hashed associative container - public: - // Though the docs say this should be num_buckets, I think it's much - // more useful as num_elements. As a special feature, calling with - // req_elements==0 will cause us to shrink if we can, saving space. - void resize(size_type req_elements) { // resize to this or larger - if ( settings.consider_shrink() || req_elements == 0 ) - maybe_shrink(); - if ( req_elements > num_elements ) - resize_delta(req_elements - num_elements); - } - - // Get and change the value of shrink_factor and enlarge_factor. The - // description at the beginning of this file explains how to choose - // the values. Setting the shrink parameter to 0.0 ensures that the - // table never shrinks. - void get_resizing_parameters(float* shrink, float* grow) const { - *shrink = settings.shrink_factor(); - *grow = settings.enlarge_factor(); - } - void set_resizing_parameters(float shrink, float grow) { - settings.set_resizing_parameters(shrink, grow); - settings.reset_thresholds(bucket_count()); - } - - // CONSTRUCTORS -- as required by the specs, we take a size, - // but also let you specify a hashfunction, key comparator, - // and key extractor. We also define a copy constructor and =. - // DESTRUCTOR -- needs to free the table - explicit dense_hashtable(size_type expected_max_items_in_table = 0, - const HashFcn& hf = HashFcn(), - const EqualKey& eql = EqualKey(), - const ExtractKey& ext = ExtractKey(), - const SetKey& set = SetKey(), - const Alloc& alloc = Alloc()) - : settings(hf), - key_info(ext, set, eql), - num_deleted(0), - num_elements(0), - num_buckets(expected_max_items_in_table == 0 - ? HT_DEFAULT_STARTING_BUCKETS - : settings.min_buckets(expected_max_items_in_table, 0)), - val_info(alloc_impl<value_alloc_type>(alloc)), - table(NULL) { - // table is NULL until emptyval is set. However, we set num_buckets - // here so we know how much space to allocate once emptyval is set - settings.reset_thresholds(bucket_count()); - } - - // As a convenience for resize(), we allow an optional second argument - // which lets you make this new hashtable a different size than ht - dense_hashtable(const dense_hashtable& ht, - size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS) - : settings(ht.settings), - key_info(ht.key_info), - num_deleted(0), - num_elements(0), - num_buckets(0), - val_info(ht.val_info), - table(NULL) { - if (!ht.settings.use_empty()) { - // If use_empty isn't set, copy_from will crash, so we do our own copying. - assert(ht.empty()); - num_buckets = settings.min_buckets(ht.size(), min_buckets_wanted); - settings.reset_thresholds(bucket_count()); - return; - } - settings.reset_thresholds(bucket_count()); - copy_from(ht, min_buckets_wanted); // copy_from() ignores deleted entries - } - - dense_hashtable& operator= (const dense_hashtable& ht) { - if (&ht == this) return *this; // don't copy onto ourselves - if (!ht.settings.use_empty()) { - assert(ht.empty()); - dense_hashtable empty_table(ht); // empty table with ht's thresholds - this->swap(empty_table); - return *this; - } - settings = ht.settings; - key_info = ht.key_info; - set_value(&val_info.emptyval, ht.val_info.emptyval); - // copy_from() calls clear and sets num_deleted to 0 too - copy_from(ht, HT_MIN_BUCKETS); - // we purposefully don't copy the allocator, which may not be copyable - return *this; - } - - ~dense_hashtable() { - if (table) { - destroy_buckets(0, num_buckets); - val_info.deallocate(table, num_buckets); - } - } - - // Many STL algorithms use swap instead of copy constructors - void swap(dense_hashtable& ht) { - std::swap(settings, ht.settings); - std::swap(key_info, ht.key_info); - std::swap(num_deleted, ht.num_deleted); - std::swap(num_elements, ht.num_elements); - std::swap(num_buckets, ht.num_buckets); - { value_type tmp; // for annoying reasons, swap() doesn't work - set_value(&tmp, val_info.emptyval); - set_value(&val_info.emptyval, ht.val_info.emptyval); - set_value(&ht.val_info.emptyval, tmp); - } - std::swap(table, ht.table); - settings.reset_thresholds(bucket_count()); // also resets consider_shrink - ht.settings.reset_thresholds(ht.bucket_count()); - // we purposefully don't swap the allocator, which may not be swap-able - } - - private: - void clear_to_size(size_type new_num_buckets) { - if (!table) { - table = val_info.allocate(new_num_buckets); - } else { - destroy_buckets(0, num_buckets); - if (new_num_buckets != num_buckets) { // resize, if necessary - typedef base::integral_constant<bool, - base::is_same<value_alloc_type, - libc_allocator_with_realloc<value_type> >::value> - realloc_ok; - resize_table(num_buckets, new_num_buckets, realloc_ok()); - } - } - assert(table); - fill_range_with_empty(table, table + new_num_buckets); - num_elements = 0; - num_deleted = 0; - num_buckets = new_num_buckets; // our new size - settings.reset_thresholds(bucket_count()); - } - - public: - // It's always nice to be able to clear a table without deallocating it - void clear() { - // If the table is already empty, and the number of buckets is - // already as we desire, there's nothing to do. - const size_type new_num_buckets = settings.min_buckets(0, 0); - if (num_elements == 0 && new_num_buckets == num_buckets) { - return; - } - clear_to_size(new_num_buckets); - } - - // Clear the table without resizing it. - // Mimicks the stl_hashtable's behaviour when clear()-ing in that it - // does not modify the bucket count - void clear_no_resize() { - if (num_elements > 0) { - assert(table); - destroy_buckets(0, num_buckets); - fill_range_with_empty(table, table + num_buckets); - } - // don't consider to shrink before another erase() - settings.reset_thresholds(bucket_count()); - num_elements = 0; - num_deleted = 0; - } - - // LOOKUP ROUTINES - private: - // Returns a pair of positions: 1st where the object is, 2nd where - // it would go if you wanted to insert it. 1st is ILLEGAL_BUCKET - // if object is not found; 2nd is ILLEGAL_BUCKET if it is. - // Note: because of deletions where-to-insert is not trivial: it's the - // first deleted bucket we see, as long as we don't find the key later - std::pair<size_type, size_type> find_position(const key_type &key) const { - size_type num_probes = 0; // how many times we've probed - const size_type bucket_count_minus_one = bucket_count() - 1; - size_type bucknum = hash(key) & bucket_count_minus_one; - size_type insert_pos = ILLEGAL_BUCKET; // where we would insert - while ( 1 ) { // probe until something happens - if ( test_empty(bucknum) ) { // bucket is empty - if ( insert_pos == ILLEGAL_BUCKET ) // found no prior place to insert - return std::pair<size_type,size_type>(ILLEGAL_BUCKET, bucknum); - else - return std::pair<size_type,size_type>(ILLEGAL_BUCKET, insert_pos); - - } else if ( test_deleted(bucknum) ) {// keep searching, but mark to insert - if ( insert_pos == ILLEGAL_BUCKET ) - insert_pos = bucknum; - - } else if ( equals(key, get_key(table[bucknum])) ) { - return std::pair<size_type,size_type>(bucknum, ILLEGAL_BUCKET); - } - ++num_probes; // we're doing another probe - bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one; - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - } - - public: - - iterator find(const key_type& key) { - if ( size() == 0 ) return end(); - std::pair<size_type, size_type> pos = find_position(key); - if ( pos.first == ILLEGAL_BUCKET ) // alas, not there - return end(); - else - return iterator(this, table + pos.first, table + num_buckets, false); - } - - const_iterator find(const key_type& key) const { - if ( size() == 0 ) return end(); - std::pair<size_type, size_type> pos = find_position(key); - if ( pos.first == ILLEGAL_BUCKET ) // alas, not there - return end(); - else - return const_iterator(this, table + pos.first, table+num_buckets, false); - } - - // This is a tr1 method: the bucket a given key is in, or what bucket - // it would be put in, if it were to be inserted. Shrug. - size_type bucket(const key_type& key) const { - std::pair<size_type, size_type> pos = find_position(key); - return pos.first == ILLEGAL_BUCKET ? pos.second : pos.first; - } - - // Counts how many elements have key key. For maps, it's either 0 or 1. - size_type count(const key_type &key) const { - std::pair<size_type, size_type> pos = find_position(key); - return pos.first == ILLEGAL_BUCKET ? 0 : 1; - } - - // Likewise, equal_range doesn't really make sense for us. Oh well. - std::pair<iterator,iterator> equal_range(const key_type& key) { - iterator pos = find(key); // either an iterator or end - if (pos == end()) { - return std::pair<iterator,iterator>(pos, pos); - } else { - const iterator startpos = pos++; - return std::pair<iterator,iterator>(startpos, pos); - } - } - std::pair<const_iterator,const_iterator> equal_range(const key_type& key) - const { - const_iterator pos = find(key); // either an iterator or end - if (pos == end()) { - return std::pair<const_iterator,const_iterator>(pos, pos); - } else { - const const_iterator startpos = pos++; - return std::pair<const_iterator,const_iterator>(startpos, pos); - } - } - - - // INSERTION ROUTINES - private: - // Private method used by insert_noresize and find_or_insert. - iterator insert_at(const_reference obj, size_type pos) { - if (size() >= max_size()) { - throw std::length_error("insert overflow"); - } - if ( test_deleted(pos) ) { // just replace if it's been del. - // shrug: shouldn't need to be const. - const_iterator delpos(this, table + pos, table + num_buckets, false); - clear_deleted(delpos); - assert( num_deleted > 0); - --num_deleted; // used to be, now it isn't - } else { - ++num_elements; // replacing an empty bucket - } - set_value(&table[pos], obj); - return iterator(this, table + pos, table + num_buckets, false); - } - - // If you know *this is big enough to hold obj, use this routine - std::pair<iterator, bool> insert_noresize(const_reference obj) { - // First, double-check we're not inserting delkey or emptyval - assert((!settings.use_empty() || !equals(get_key(obj), - get_key(val_info.emptyval))) - && "Inserting the empty key"); - assert((!settings.use_deleted() || !equals(get_key(obj), key_info.delkey)) - && "Inserting the deleted key"); - const std::pair<size_type,size_type> pos = find_position(get_key(obj)); - if ( pos.first != ILLEGAL_BUCKET) { // object was already there - return std::pair<iterator,bool>(iterator(this, table + pos.first, - table + num_buckets, false), - false); // false: we didn't insert - } else { // pos.second says where to put it - return std::pair<iterator,bool>(insert_at(obj, pos.second), true); - } - } - - // Specializations of insert(it, it) depending on the power of the iterator: - // (1) Iterator supports operator-, resize before inserting - template <class ForwardIterator> - void insert(ForwardIterator f, ForwardIterator l, std::forward_iterator_tag) { - size_t dist = std::distance(f, l); - if (dist >= (std::numeric_limits<size_type>::max)()) { - throw std::length_error("insert-range overflow"); - } - resize_delta(static_cast<size_type>(dist)); - for ( ; dist > 0; --dist, ++f) { - insert_noresize(*f); - } - } - - // (2) Arbitrary iterator, can't tell how much to resize - template <class InputIterator> - void insert(InputIterator f, InputIterator l, std::input_iterator_tag) { - for ( ; f != l; ++f) - insert(*f); - } - - public: - // This is the normal insert routine, used by the outside world - std::pair<iterator, bool> insert(const_reference obj) { - resize_delta(1); // adding an object, grow if need be - return insert_noresize(obj); - } - - // When inserting a lot at a time, we specialize on the type of iterator - template <class InputIterator> - void insert(InputIterator f, InputIterator l) { - // specializes on iterator type - insert(f, l, - typename std::iterator_traits<InputIterator>::iterator_category()); - } - - // DefaultValue is a functor that takes a key and returns a value_type - // representing the default value to be inserted if none is found. - template <class DefaultValue> - value_type& find_or_insert(const key_type& key) { - // First, double-check we're not inserting emptykey or delkey - assert((!settings.use_empty() || !equals(key, get_key(val_info.emptyval))) - && "Inserting the empty key"); - assert((!settings.use_deleted() || !equals(key, key_info.delkey)) - && "Inserting the deleted key"); - const std::pair<size_type,size_type> pos = find_position(key); - DefaultValue default_value; - if ( pos.first != ILLEGAL_BUCKET) { // object was already there - return table[pos.first]; - } else if (resize_delta(1)) { // needed to rehash to make room - // Since we resized, we can't use pos, so recalculate where to insert. - return *insert_noresize(default_value(key)).first; - } else { // no need to rehash, insert right here - return *insert_at(default_value(key), pos.second); - } - } - - - // DELETION ROUTINES - size_type erase(const key_type& key) { - // First, double-check we're not trying to erase delkey or emptyval. - assert((!settings.use_empty() || !equals(key, get_key(val_info.emptyval))) - && "Erasing the empty key"); - assert((!settings.use_deleted() || !equals(key, key_info.delkey)) - && "Erasing the deleted key"); - const_iterator pos = find(key); // shrug: shouldn't need to be const - if ( pos != end() ) { - assert(!test_deleted(pos)); // or find() shouldn't have returned it - set_deleted(pos); - ++num_deleted; - settings.set_consider_shrink(true); // will think about shrink after next insert - return 1; // because we deleted one thing - } else { - return 0; // because we deleted nothing - } - } - - // We return the iterator past the deleted item. - void erase(iterator pos) { - if ( pos == end() ) return; // sanity check - if ( set_deleted(pos) ) { // true if object has been newly deleted - ++num_deleted; - settings.set_consider_shrink(true); // will think about shrink after next insert - } - } - - void erase(iterator f, iterator l) { - for ( ; f != l; ++f) { - if ( set_deleted(f) ) // should always be true - ++num_deleted; - } - settings.set_consider_shrink(true); // will think about shrink after next insert - } - - // We allow you to erase a const_iterator just like we allow you to - // erase an iterator. This is in parallel to 'delete': you can delete - // a const pointer just like a non-const pointer. The logic is that - // you can't use the object after it's erased anyway, so it doesn't matter - // if it's const or not. - void erase(const_iterator pos) { - if ( pos == end() ) return; // sanity check - if ( set_deleted(pos) ) { // true if object has been newly deleted - ++num_deleted; - settings.set_consider_shrink(true); // will think about shrink after next insert - } - } - void erase(const_iterator f, const_iterator l) { - for ( ; f != l; ++f) { - if ( set_deleted(f) ) // should always be true - ++num_deleted; - } - settings.set_consider_shrink(true); // will think about shrink after next insert - } - - - // COMPARISON - bool operator==(const dense_hashtable& ht) const { - if (size() != ht.size()) { - return false; - } else if (this == &ht) { - return true; - } else { - // Iterate through the elements in "this" and see if the - // corresponding element is in ht - for ( const_iterator it = begin(); it != end(); ++it ) { - const_iterator it2 = ht.find(get_key(*it)); - if ((it2 == ht.end()) || (*it != *it2)) { - return false; - } - } - return true; - } - } - bool operator!=(const dense_hashtable& ht) const { - return !(*this == ht); - } - - - // I/O - // We support reading and writing hashtables to disk. Alas, since - // I don't know how to write a hasher or key_equal, you have to make - // sure everything but the table is the same. We compact before writing. - private: - // Every time the disk format changes, this should probably change too - typedef unsigned long MagicNumberType; - static const MagicNumberType MAGIC_NUMBER = 0x13578642; - - public: - // I/O -- this is an add-on for writing hash table to disk - // - // INPUT and OUTPUT must be either a FILE, *or* a C++ stream - // (istream, ostream, etc) *or* a class providing - // Read(void*, size_t) and Write(const void*, size_t) - // (respectively), which writes a buffer into a stream - // (which the INPUT/OUTPUT instance presumably owns). - - typedef sparsehash_internal::pod_serializer<value_type> NopointerSerializer; - - // ValueSerializer: a functor. operator()(OUTPUT*, const value_type&) - template <typename ValueSerializer, typename OUTPUT> - bool serialize(ValueSerializer serializer, OUTPUT *fp) { - squash_deleted(); // so we don't have to worry about delkey - if ( !sparsehash_internal::write_bigendian_number(fp, MAGIC_NUMBER, 4) ) - return false; - if ( !sparsehash_internal::write_bigendian_number(fp, num_buckets, 8) ) - return false; - if ( !sparsehash_internal::write_bigendian_number(fp, num_elements, 8) ) - return false; - // Now write a bitmap of non-empty buckets. - for ( size_type i = 0; i < num_buckets; i += 8 ) { - unsigned char bits = 0; - for ( int bit = 0; bit < 8; ++bit ) { - if ( i + bit < num_buckets && !test_empty(i + bit) ) - bits |= (1 << bit); - } - if ( !sparsehash_internal::write_data(fp, &bits, sizeof(bits)) ) - return false; - for ( int bit = 0; bit < 8; ++bit ) { - if ( bits & (1 << bit) ) { - if ( !serializer(fp, table[i + bit]) ) return false; - } - } - } - return true; - } - - // INPUT: anything we've written an overload of read_data() for. - // ValueSerializer: a functor. operator()(INPUT*, value_type*) - template <typename ValueSerializer, typename INPUT> - bool unserialize(ValueSerializer serializer, INPUT *fp) { - assert(settings.use_empty() && "empty_key not set for read"); - - clear(); // just to be consistent - MagicNumberType magic_read; - if ( !sparsehash_internal::read_bigendian_number(fp, &magic_read, 4) ) - return false; - if ( magic_read != MAGIC_NUMBER ) { - return false; - } - size_type new_num_buckets; - if ( !sparsehash_internal::read_bigendian_number(fp, &new_num_buckets, 8) ) - return false; - clear_to_size(new_num_buckets); - if ( !sparsehash_internal::read_bigendian_number(fp, &num_elements, 8) ) - return false; - - // Read the bitmap of non-empty buckets. - for (size_type i = 0; i < num_buckets; i += 8) { - unsigned char bits; - if ( !sparsehash_internal::read_data(fp, &bits, sizeof(bits)) ) - return false; - for ( int bit = 0; bit < 8; ++bit ) { - if ( i + bit < num_buckets && (bits & (1 << bit)) ) { // not empty - if ( !serializer(fp, &table[i + bit]) ) return false; - } - } - } - return true; - } - - private: - template <class A> - class alloc_impl : public A { - public: - typedef typename A::pointer pointer; - typedef typename A::size_type size_type; - - // Convert a normal allocator to one that has realloc_or_die() - alloc_impl(const A& a) : A(a) { } - - // realloc_or_die should only be used when using the default - // allocator (libc_allocator_with_realloc). - pointer realloc_or_die(pointer /*ptr*/, size_type /*n*/) { - fprintf(stderr, "realloc_or_die is only supported for " - "libc_allocator_with_realloc\n"); - exit(1); - return NULL; - } - }; - - // A template specialization of alloc_impl for - // libc_allocator_with_realloc that can handle realloc_or_die. - template <class A> - class alloc_impl<libc_allocator_with_realloc<A> > - : public libc_allocator_with_realloc<A> { - public: - typedef typename libc_allocator_with_realloc<A>::pointer pointer; - typedef typename libc_allocator_with_realloc<A>::size_type size_type; - - alloc_impl(const libc_allocator_with_realloc<A>& a) - : libc_allocator_with_realloc<A>(a) { } - - pointer realloc_or_die(pointer ptr, size_type n) { - pointer retval = this->reallocate(ptr, n); - if (retval == NULL) { + if (resize_to < needed_size && // may double resize_to + resize_to < (std::numeric_limits<size_type>::max)() / 2) { + // This situation means that we have enough deleted elements, + // that once we purge them, we won't actually have needed to + // grow. But we may want to grow anyway: if we just purge one + // element, say, we'll have to grow anyway next time we + // insert. Might as well grow now, since we're already going + // through the trouble of copying (in order to purge the + // deleted elements). + const size_type target = + static_cast<size_type>(settings.shrink_size(resize_to*2)); + if (num_elements - num_deleted + delta >= target) { + // Good, we won't be below the shrink threshhold even if we double. + resize_to *= 2; + } + } + dense_hashtable tmp(*this, resize_to); + swap(tmp); // now we are tmp + return true; + } + + // We require table be not-NULL and empty before calling this. + void resize_table(size_type /*old_size*/, size_type new_size, + base::true_type) { + table = val_info.realloc_or_die(table, new_size); + } + + void resize_table(size_type old_size, size_type new_size, base::false_type) { + val_info.deallocate(table, old_size); + table = val_info.allocate(new_size); + } + + // Used to actually do the rehashing when we grow/shrink a hashtable + void copy_from(const dense_hashtable &ht, size_type min_buckets_wanted) { + clear_to_size(settings.min_buckets(ht.size(), min_buckets_wanted)); + + // We use a normal iterator to get non-deleted bcks from ht + // We could use insert() here, but since we know there are + // no duplicates and no deleted items, we can be more efficient + assert((bucket_count() & (bucket_count()-1)) == 0); // a power of two + for ( const_iterator it = ht.begin(); it != ht.end(); ++it ) { + size_type num_probes = 0; // how many times we've probed + size_type bucknum; + const size_type bucket_count_minus_one = bucket_count() - 1; + for (bucknum = hash(get_key(*it)) & bucket_count_minus_one; + !test_empty(bucknum); // not empty + bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one) { + ++num_probes; + assert(num_probes < bucket_count() + && "Hashtable is full: an error in key_equal<> or hash<>"); + } + set_value(&table[bucknum], *it); // copies the value to here + num_elements++; + } + settings.inc_num_ht_copies(); + } + + // Required by the spec for hashed associative container + public: + // Though the docs say this should be num_buckets, I think it's much + // more useful as num_elements. As a special feature, calling with + // req_elements==0 will cause us to shrink if we can, saving space. + void resize(size_type req_elements) { // resize to this or larger + if ( settings.consider_shrink() || req_elements == 0 ) + maybe_shrink(); + if ( req_elements > num_elements ) + resize_delta(req_elements - num_elements); + } + + // Get and change the value of shrink_factor and enlarge_factor. The + // description at the beginning of this file explains how to choose + // the values. Setting the shrink parameter to 0.0 ensures that the + // table never shrinks. + void get_resizing_parameters(float* shrink, float* grow) const { + *shrink = settings.shrink_factor(); + *grow = settings.enlarge_factor(); + } + void set_resizing_parameters(float shrink, float grow) { + settings.set_resizing_parameters(shrink, grow); + settings.reset_thresholds(bucket_count()); + } + + // CONSTRUCTORS -- as required by the specs, we take a size, + // but also let you specify a hashfunction, key comparator, + // and key extractor. We also define a copy constructor and =. + // DESTRUCTOR -- needs to free the table + explicit dense_hashtable(size_type expected_max_items_in_table = 0, + const HashFcn& hf = HashFcn(), + const EqualKey& eql = EqualKey(), + const ExtractKey& ext = ExtractKey(), + const SetKey& set = SetKey(), + const Alloc& alloc = Alloc()) + : settings(hf), + key_info(ext, set, eql), + num_deleted(0), + num_elements(0), + num_buckets(expected_max_items_in_table == 0 + ? HT_DEFAULT_STARTING_BUCKETS + : settings.min_buckets(expected_max_items_in_table, 0)), + val_info(alloc_impl<value_alloc_type>(alloc)), + table(NULL) { + // table is NULL until emptyval is set. However, we set num_buckets + // here so we know how much space to allocate once emptyval is set + settings.reset_thresholds(bucket_count()); + } + + // As a convenience for resize(), we allow an optional second argument + // which lets you make this new hashtable a different size than ht + dense_hashtable(const dense_hashtable& ht, + size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS) + : settings(ht.settings), + key_info(ht.key_info), + num_deleted(0), + num_elements(0), + num_buckets(0), + val_info(ht.val_info), + table(NULL) { + if (!ht.settings.use_empty()) { + // If use_empty isn't set, copy_from will crash, so we do our own copying. + assert(ht.empty()); + num_buckets = settings.min_buckets(ht.size(), min_buckets_wanted); + settings.reset_thresholds(bucket_count()); + return; + } + settings.reset_thresholds(bucket_count()); + copy_from(ht, min_buckets_wanted); // copy_from() ignores deleted entries + } + + dense_hashtable& operator= (const dense_hashtable& ht) { + if (&ht == this) return *this; // don't copy onto ourselves + if (!ht.settings.use_empty()) { + assert(ht.empty()); + dense_hashtable empty_table(ht); // empty table with ht's thresholds + this->swap(empty_table); + return *this; + } + settings = ht.settings; + key_info = ht.key_info; + set_value(&val_info.emptyval, ht.val_info.emptyval); + // copy_from() calls clear and sets num_deleted to 0 too + copy_from(ht, HT_MIN_BUCKETS); + // we purposefully don't copy the allocator, which may not be copyable + return *this; + } + + ~dense_hashtable() { + if (table) { + destroy_buckets(0, num_buckets); + val_info.deallocate(table, num_buckets); + } + } + + // Many STL algorithms use swap instead of copy constructors + void swap(dense_hashtable& ht) { + std::swap(settings, ht.settings); + std::swap(key_info, ht.key_info); + std::swap(num_deleted, ht.num_deleted); + std::swap(num_elements, ht.num_elements); + std::swap(num_buckets, ht.num_buckets); + { value_type tmp; // for annoying reasons, swap() doesn't work + set_value(&tmp, val_info.emptyval); + set_value(&val_info.emptyval, ht.val_info.emptyval); + set_value(&ht.val_info.emptyval, tmp); + } + std::swap(table, ht.table); + settings.reset_thresholds(bucket_count()); // also resets consider_shrink + ht.settings.reset_thresholds(ht.bucket_count()); + // we purposefully don't swap the allocator, which may not be swap-able + } + + private: + void clear_to_size(size_type new_num_buckets) { + if (!table) { + table = val_info.allocate(new_num_buckets); + } else { + destroy_buckets(0, num_buckets); + if (new_num_buckets != num_buckets) { // resize, if necessary + typedef base::integral_constant<bool, + base::is_same<value_alloc_type, + libc_allocator_with_realloc<value_type> >::value> + realloc_ok; + resize_table(num_buckets, new_num_buckets, realloc_ok()); + } + } + assert(table); + fill_range_with_empty(table, table + new_num_buckets); + num_elements = 0; + num_deleted = 0; + num_buckets = new_num_buckets; // our new size + settings.reset_thresholds(bucket_count()); + } + + public: + // It's always nice to be able to clear a table without deallocating it + void clear() { + // If the table is already empty, and the number of buckets is + // already as we desire, there's nothing to do. + const size_type new_num_buckets = settings.min_buckets(0, 0); + if (num_elements == 0 && new_num_buckets == num_buckets) { + return; + } + clear_to_size(new_num_buckets); + } + + // Clear the table without resizing it. + // Mimicks the stl_hashtable's behaviour when clear()-ing in that it + // does not modify the bucket count + void clear_no_resize() { + if (num_elements > 0) { + assert(table); + destroy_buckets(0, num_buckets); + fill_range_with_empty(table, table + num_buckets); + } + // don't consider to shrink before another erase() + settings.reset_thresholds(bucket_count()); + num_elements = 0; + num_deleted = 0; + } + + // LOOKUP ROUTINES + private: + // Returns a pair of positions: 1st where the object is, 2nd where + // it would go if you wanted to insert it. 1st is ILLEGAL_BUCKET + // if object is not found; 2nd is ILLEGAL_BUCKET if it is. + // Note: because of deletions where-to-insert is not trivial: it's the + // first deleted bucket we see, as long as we don't find the key later + std::pair<size_type, size_type> find_position(const key_type &key) const { + size_type num_probes = 0; // how many times we've probed + const size_type bucket_count_minus_one = bucket_count() - 1; + size_type bucknum = hash(key) & bucket_count_minus_one; + size_type insert_pos = ILLEGAL_BUCKET; // where we would insert + while ( 1 ) { // probe until something happens + if ( test_empty(bucknum) ) { // bucket is empty + if ( insert_pos == ILLEGAL_BUCKET ) // found no prior place to insert + return std::pair<size_type,size_type>(ILLEGAL_BUCKET, bucknum); + else + return std::pair<size_type,size_type>(ILLEGAL_BUCKET, insert_pos); + + } else if ( test_deleted(bucknum) ) {// keep searching, but mark to insert + if ( insert_pos == ILLEGAL_BUCKET ) + insert_pos = bucknum; + + } else if ( equals(key, get_key(table[bucknum])) ) { + return std::pair<size_type,size_type>(bucknum, ILLEGAL_BUCKET); + } + ++num_probes; // we're doing another probe + bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one; + assert(num_probes < bucket_count() + && "Hashtable is full: an error in key_equal<> or hash<>"); + } + } + + public: + + iterator find(const key_type& key) { + if ( size() == 0 ) return end(); + std::pair<size_type, size_type> pos = find_position(key); + if ( pos.first == ILLEGAL_BUCKET ) // alas, not there + return end(); + else + return iterator(this, table + pos.first, table + num_buckets, false); + } + + const_iterator find(const key_type& key) const { + if ( size() == 0 ) return end(); + std::pair<size_type, size_type> pos = find_position(key); + if ( pos.first == ILLEGAL_BUCKET ) // alas, not there + return end(); + else + return const_iterator(this, table + pos.first, table+num_buckets, false); + } + + // This is a tr1 method: the bucket a given key is in, or what bucket + // it would be put in, if it were to be inserted. Shrug. + size_type bucket(const key_type& key) const { + std::pair<size_type, size_type> pos = find_position(key); + return pos.first == ILLEGAL_BUCKET ? pos.second : pos.first; + } + + // Counts how many elements have key key. For maps, it's either 0 or 1. + size_type count(const key_type &key) const { + std::pair<size_type, size_type> pos = find_position(key); + return pos.first == ILLEGAL_BUCKET ? 0 : 1; + } + + // Likewise, equal_range doesn't really make sense for us. Oh well. + std::pair<iterator,iterator> equal_range(const key_type& key) { + iterator pos = find(key); // either an iterator or end + if (pos == end()) { + return std::pair<iterator,iterator>(pos, pos); + } else { + const iterator startpos = pos++; + return std::pair<iterator,iterator>(startpos, pos); + } + } + std::pair<const_iterator,const_iterator> equal_range(const key_type& key) + const { + const_iterator pos = find(key); // either an iterator or end + if (pos == end()) { + return std::pair<const_iterator,const_iterator>(pos, pos); + } else { + const const_iterator startpos = pos++; + return std::pair<const_iterator,const_iterator>(startpos, pos); + } + } + + + // INSERTION ROUTINES + private: + // Private method used by insert_noresize and find_or_insert. + iterator insert_at(const_reference obj, size_type pos) { + if (size() >= max_size()) { + throw std::length_error("insert overflow"); + } + if ( test_deleted(pos) ) { // just replace if it's been del. + // shrug: shouldn't need to be const. + const_iterator delpos(this, table + pos, table + num_buckets, false); + clear_deleted(delpos); + assert( num_deleted > 0); + --num_deleted; // used to be, now it isn't + } else { + ++num_elements; // replacing an empty bucket + } + set_value(&table[pos], obj); + return iterator(this, table + pos, table + num_buckets, false); + } + + // If you know *this is big enough to hold obj, use this routine + std::pair<iterator, bool> insert_noresize(const_reference obj) { + // First, double-check we're not inserting delkey or emptyval + assert((!settings.use_empty() || !equals(get_key(obj), + get_key(val_info.emptyval))) + && "Inserting the empty key"); + assert((!settings.use_deleted() || !equals(get_key(obj), key_info.delkey)) + && "Inserting the deleted key"); + const std::pair<size_type,size_type> pos = find_position(get_key(obj)); + if ( pos.first != ILLEGAL_BUCKET) { // object was already there + return std::pair<iterator,bool>(iterator(this, table + pos.first, + table + num_buckets, false), + false); // false: we didn't insert + } else { // pos.second says where to put it + return std::pair<iterator,bool>(insert_at(obj, pos.second), true); + } + } + + // Specializations of insert(it, it) depending on the power of the iterator: + // (1) Iterator supports operator-, resize before inserting + template <class ForwardIterator> + void insert(ForwardIterator f, ForwardIterator l, std::forward_iterator_tag) { + size_t dist = std::distance(f, l); + if (dist >= (std::numeric_limits<size_type>::max)()) { + throw std::length_error("insert-range overflow"); + } + resize_delta(static_cast<size_type>(dist)); + for ( ; dist > 0; --dist, ++f) { + insert_noresize(*f); + } + } + + // (2) Arbitrary iterator, can't tell how much to resize + template <class InputIterator> + void insert(InputIterator f, InputIterator l, std::input_iterator_tag) { + for ( ; f != l; ++f) + insert(*f); + } + + public: + // This is the normal insert routine, used by the outside world + std::pair<iterator, bool> insert(const_reference obj) { + resize_delta(1); // adding an object, grow if need be + return insert_noresize(obj); + } + + // When inserting a lot at a time, we specialize on the type of iterator + template <class InputIterator> + void insert(InputIterator f, InputIterator l) { + // specializes on iterator type + insert(f, l, + typename std::iterator_traits<InputIterator>::iterator_category()); + } + + // DefaultValue is a functor that takes a key and returns a value_type + // representing the default value to be inserted if none is found. + template <class DefaultValue> + value_type& find_or_insert(const key_type& key) { + // First, double-check we're not inserting emptykey or delkey + assert((!settings.use_empty() || !equals(key, get_key(val_info.emptyval))) + && "Inserting the empty key"); + assert((!settings.use_deleted() || !equals(key, key_info.delkey)) + && "Inserting the deleted key"); + const std::pair<size_type,size_type> pos = find_position(key); + DefaultValue default_value; + if ( pos.first != ILLEGAL_BUCKET) { // object was already there + return table[pos.first]; + } else if (resize_delta(1)) { // needed to rehash to make room + // Since we resized, we can't use pos, so recalculate where to insert. + return *insert_noresize(default_value(key)).first; + } else { // no need to rehash, insert right here + return *insert_at(default_value(key), pos.second); + } + } + + + // DELETION ROUTINES + size_type erase(const key_type& key) { + // First, double-check we're not trying to erase delkey or emptyval. + assert((!settings.use_empty() || !equals(key, get_key(val_info.emptyval))) + && "Erasing the empty key"); + assert((!settings.use_deleted() || !equals(key, key_info.delkey)) + && "Erasing the deleted key"); + const_iterator pos = find(key); // shrug: shouldn't need to be const + if ( pos != end() ) { + assert(!test_deleted(pos)); // or find() shouldn't have returned it + set_deleted(pos); + ++num_deleted; + settings.set_consider_shrink(true); // will think about shrink after next insert + return 1; // because we deleted one thing + } else { + return 0; // because we deleted nothing + } + } + + // We return the iterator past the deleted item. + void erase(iterator pos) { + if ( pos == end() ) return; // sanity check + if ( set_deleted(pos) ) { // true if object has been newly deleted + ++num_deleted; + settings.set_consider_shrink(true); // will think about shrink after next insert + } + } + + void erase(iterator f, iterator l) { + for ( ; f != l; ++f) { + if ( set_deleted(f) ) // should always be true + ++num_deleted; + } + settings.set_consider_shrink(true); // will think about shrink after next insert + } + + // We allow you to erase a const_iterator just like we allow you to + // erase an iterator. This is in parallel to 'delete': you can delete + // a const pointer just like a non-const pointer. The logic is that + // you can't use the object after it's erased anyway, so it doesn't matter + // if it's const or not. + void erase(const_iterator pos) { + if ( pos == end() ) return; // sanity check + if ( set_deleted(pos) ) { // true if object has been newly deleted + ++num_deleted; + settings.set_consider_shrink(true); // will think about shrink after next insert + } + } + void erase(const_iterator f, const_iterator l) { + for ( ; f != l; ++f) { + if ( set_deleted(f) ) // should always be true + ++num_deleted; + } + settings.set_consider_shrink(true); // will think about shrink after next insert + } + + + // COMPARISON + bool operator==(const dense_hashtable& ht) const { + if (size() != ht.size()) { + return false; + } else if (this == &ht) { + return true; + } else { + // Iterate through the elements in "this" and see if the + // corresponding element is in ht + for ( const_iterator it = begin(); it != end(); ++it ) { + const_iterator it2 = ht.find(get_key(*it)); + if ((it2 == ht.end()) || (*it != *it2)) { + return false; + } + } + return true; + } + } + bool operator!=(const dense_hashtable& ht) const { + return !(*this == ht); + } + + + // I/O + // We support reading and writing hashtables to disk. Alas, since + // I don't know how to write a hasher or key_equal, you have to make + // sure everything but the table is the same. We compact before writing. + private: + // Every time the disk format changes, this should probably change too + typedef unsigned long MagicNumberType; + static const MagicNumberType MAGIC_NUMBER = 0x13578642; + + public: + // I/O -- this is an add-on for writing hash table to disk + // + // INPUT and OUTPUT must be either a FILE, *or* a C++ stream + // (istream, ostream, etc) *or* a class providing + // Read(void*, size_t) and Write(const void*, size_t) + // (respectively), which writes a buffer into a stream + // (which the INPUT/OUTPUT instance presumably owns). + + typedef sparsehash_internal::pod_serializer<value_type> NopointerSerializer; + + // ValueSerializer: a functor. operator()(OUTPUT*, const value_type&) + template <typename ValueSerializer, typename OUTPUT> + bool serialize(ValueSerializer serializer, OUTPUT *fp) { + squash_deleted(); // so we don't have to worry about delkey + if ( !sparsehash_internal::write_bigendian_number(fp, MAGIC_NUMBER, 4) ) + return false; + if ( !sparsehash_internal::write_bigendian_number(fp, num_buckets, 8) ) + return false; + if ( !sparsehash_internal::write_bigendian_number(fp, num_elements, 8) ) + return false; + // Now write a bitmap of non-empty buckets. + for ( size_type i = 0; i < num_buckets; i += 8 ) { + unsigned char bits = 0; + for ( int bit = 0; bit < 8; ++bit ) { + if ( i + bit < num_buckets && !test_empty(i + bit) ) + bits |= (1 << bit); + } + if ( !sparsehash_internal::write_data(fp, &bits, sizeof(bits)) ) + return false; + for ( int bit = 0; bit < 8; ++bit ) { + if ( bits & (1 << bit) ) { + if ( !serializer(fp, table[i + bit]) ) return false; + } + } + } + return true; + } + + // INPUT: anything we've written an overload of read_data() for. + // ValueSerializer: a functor. operator()(INPUT*, value_type*) + template <typename ValueSerializer, typename INPUT> + bool unserialize(ValueSerializer serializer, INPUT *fp) { + assert(settings.use_empty() && "empty_key not set for read"); + + clear(); // just to be consistent + MagicNumberType magic_read; + if ( !sparsehash_internal::read_bigendian_number(fp, &magic_read, 4) ) + return false; + if ( magic_read != MAGIC_NUMBER ) { + return false; + } + size_type new_num_buckets; + if ( !sparsehash_internal::read_bigendian_number(fp, &new_num_buckets, 8) ) + return false; + clear_to_size(new_num_buckets); + if ( !sparsehash_internal::read_bigendian_number(fp, &num_elements, 8) ) + return false; + + // Read the bitmap of non-empty buckets. + for (size_type i = 0; i < num_buckets; i += 8) { + unsigned char bits; + if ( !sparsehash_internal::read_data(fp, &bits, sizeof(bits)) ) + return false; + for ( int bit = 0; bit < 8; ++bit ) { + if ( i + bit < num_buckets && (bits & (1 << bit)) ) { // not empty + if ( !serializer(fp, &table[i + bit]) ) return false; + } + } + } + return true; + } + + private: + template <class A> + class alloc_impl : public A { + public: + typedef typename A::pointer pointer; + typedef typename A::size_type size_type; + + // Convert a normal allocator to one that has realloc_or_die() + alloc_impl(const A& a) : A(a) { } + + // realloc_or_die should only be used when using the default + // allocator (libc_allocator_with_realloc). + pointer realloc_or_die(pointer /*ptr*/, size_type /*n*/) { + fprintf(stderr, "realloc_or_die is only supported for " + "libc_allocator_with_realloc\n"); + exit(1); + return NULL; + } + }; + + // A template specialization of alloc_impl for + // libc_allocator_with_realloc that can handle realloc_or_die. + template <class A> + class alloc_impl<libc_allocator_with_realloc<A> > + : public libc_allocator_with_realloc<A> { + public: + typedef typename libc_allocator_with_realloc<A>::pointer pointer; + typedef typename libc_allocator_with_realloc<A>::size_type size_type; + + alloc_impl(const libc_allocator_with_realloc<A>& a) + : libc_allocator_with_realloc<A>(a) { } + + pointer realloc_or_die(pointer ptr, size_type n) { + pointer retval = this->reallocate(ptr, n); + if (retval == NULL) { fprintf(stderr, "sparsehash: FATAL ERROR: failed to reallocate " "%lu elements for ptr %p", static_cast<unsigned long>(n), static_cast<void*>(ptr)); - exit(1); - } - return retval; - } - }; - - // Package allocator with emptyval to eliminate memory needed for - // the zero-size allocator. - // If new fields are added to this class, we should add them to - // operator= and swap. - class ValInfo : public alloc_impl<value_alloc_type> { - public: - typedef typename alloc_impl<value_alloc_type>::value_type value_type; - - ValInfo(const alloc_impl<value_alloc_type>& a) - : alloc_impl<value_alloc_type>(a), emptyval() { } - ValInfo(const ValInfo& v) - : alloc_impl<value_alloc_type>(v), emptyval(v.emptyval) { } - - value_type emptyval; // which key marks unused entries - }; - - - // Package functors with another class to eliminate memory needed for - // zero-size functors. Since ExtractKey and hasher's operator() might - // have the same function signature, they must be packaged in - // different classes. - struct Settings : - sparsehash_internal::sh_hashtable_settings<key_type, hasher, - size_type, HT_MIN_BUCKETS> { - explicit Settings(const hasher& hf) - : sparsehash_internal::sh_hashtable_settings<key_type, hasher, - size_type, HT_MIN_BUCKETS>( - hf, HT_OCCUPANCY_PCT / 100.0f, HT_EMPTY_PCT / 100.0f) {} - }; - - // Packages ExtractKey and SetKey functors. - class KeyInfo : public ExtractKey, public SetKey, public EqualKey { - public: - KeyInfo(const ExtractKey& ek, const SetKey& sk, const EqualKey& eq) - : ExtractKey(ek), - SetKey(sk), - EqualKey(eq) { - } - - // We want to return the exact same type as ExtractKey: Key or const Key& - typename ExtractKey::result_type get_key(const_reference v) const { - return ExtractKey::operator()(v); - } - void set_key(pointer v, const key_type& k) const { - SetKey::operator()(v, k); - } - bool equals(const key_type& a, const key_type& b) const { - return EqualKey::operator()(a, b); - } - - // Which key marks deleted entries. - // TODO(csilvers): make a pointer, and get rid of use_deleted (benchmark!) - typename base::remove_const<key_type>::type delkey; - }; - - // Utility functions to access the templated operators - size_type hash(const key_type& v) const { - return settings.hash(v); - } - bool equals(const key_type& a, const key_type& b) const { - return key_info.equals(a, b); - } - typename ExtractKey::result_type get_key(const_reference v) const { - return key_info.get_key(v); - } - void set_key(pointer v, const key_type& k) const { - key_info.set_key(v, k); - } - - private: - // Actual data - Settings settings; - KeyInfo key_info; - - size_type num_deleted; // how many occupied buckets are marked deleted - size_type num_elements; - size_type num_buckets; - ValInfo val_info; // holds emptyval, and also the allocator - pointer table; -}; - - -// We need a global swap as well -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -inline void swap(dense_hashtable<V,K,HF,ExK,SetK,EqK,A> &x, - dense_hashtable<V,K,HF,ExK,SetK,EqK,A> &y) { - x.swap(y); -} - -#undef JUMP_ - -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -const typename dense_hashtable<V,K,HF,ExK,SetK,EqK,A>::size_type - dense_hashtable<V,K,HF,ExK,SetK,EqK,A>::ILLEGAL_BUCKET; - -// How full we let the table get before we resize. Knuth says .8 is -// good -- higher causes us to probe too much, though saves memory. -// However, we go with .5, getting better performance at the cost of -// more space (a trade-off densehashtable explicitly chooses to make). -// Feel free to play around with different values, though, via -// max_load_factor() and/or set_resizing_parameters(). -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -const int dense_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT = 50; - -// How empty we let the table get before we resize lower. -// It should be less than OCCUPANCY_PCT / 2 or we thrash resizing. -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -const int dense_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_EMPTY_PCT - = static_cast<int>(0.4 * - dense_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT); - -_END_GOOGLE_NAMESPACE_ - -#endif /* _DENSEHASHTABLE_H_ */ + exit(1); + } + return retval; + } + }; + + // Package allocator with emptyval to eliminate memory needed for + // the zero-size allocator. + // If new fields are added to this class, we should add them to + // operator= and swap. + class ValInfo : public alloc_impl<value_alloc_type> { + public: + typedef typename alloc_impl<value_alloc_type>::value_type value_type; + + ValInfo(const alloc_impl<value_alloc_type>& a) + : alloc_impl<value_alloc_type>(a), emptyval() { } + ValInfo(const ValInfo& v) + : alloc_impl<value_alloc_type>(v), emptyval(v.emptyval) { } + + value_type emptyval; // which key marks unused entries + }; + + + // Package functors with another class to eliminate memory needed for + // zero-size functors. Since ExtractKey and hasher's operator() might + // have the same function signature, they must be packaged in + // different classes. + struct Settings : + sparsehash_internal::sh_hashtable_settings<key_type, hasher, + size_type, HT_MIN_BUCKETS> { + explicit Settings(const hasher& hf) + : sparsehash_internal::sh_hashtable_settings<key_type, hasher, + size_type, HT_MIN_BUCKETS>( + hf, HT_OCCUPANCY_PCT / 100.0f, HT_EMPTY_PCT / 100.0f) {} + }; + + // Packages ExtractKey and SetKey functors. + class KeyInfo : public ExtractKey, public SetKey, public EqualKey { + public: + KeyInfo(const ExtractKey& ek, const SetKey& sk, const EqualKey& eq) + : ExtractKey(ek), + SetKey(sk), + EqualKey(eq) { + } + + // We want to return the exact same type as ExtractKey: Key or const Key& + typename ExtractKey::result_type get_key(const_reference v) const { + return ExtractKey::operator()(v); + } + void set_key(pointer v, const key_type& k) const { + SetKey::operator()(v, k); + } + bool equals(const key_type& a, const key_type& b) const { + return EqualKey::operator()(a, b); + } + + // Which key marks deleted entries. + // TODO(csilvers): make a pointer, and get rid of use_deleted (benchmark!) + typename base::remove_const<key_type>::type delkey; + }; + + // Utility functions to access the templated operators + size_type hash(const key_type& v) const { + return settings.hash(v); + } + bool equals(const key_type& a, const key_type& b) const { + return key_info.equals(a, b); + } + typename ExtractKey::result_type get_key(const_reference v) const { + return key_info.get_key(v); + } + void set_key(pointer v, const key_type& k) const { + key_info.set_key(v, k); + } + + private: + // Actual data + Settings settings; + KeyInfo key_info; + + size_type num_deleted; // how many occupied buckets are marked deleted + size_type num_elements; + size_type num_buckets; + ValInfo val_info; // holds emptyval, and also the allocator + pointer table; +}; + + +// We need a global swap as well +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +inline void swap(dense_hashtable<V,K,HF,ExK,SetK,EqK,A> &x, + dense_hashtable<V,K,HF,ExK,SetK,EqK,A> &y) { + x.swap(y); +} + +#undef JUMP_ + +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +const typename dense_hashtable<V,K,HF,ExK,SetK,EqK,A>::size_type + dense_hashtable<V,K,HF,ExK,SetK,EqK,A>::ILLEGAL_BUCKET; + +// How full we let the table get before we resize. Knuth says .8 is +// good -- higher causes us to probe too much, though saves memory. +// However, we go with .5, getting better performance at the cost of +// more space (a trade-off densehashtable explicitly chooses to make). +// Feel free to play around with different values, though, via +// max_load_factor() and/or set_resizing_parameters(). +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +const int dense_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT = 50; + +// How empty we let the table get before we resize lower. +// It should be less than OCCUPANCY_PCT / 2 or we thrash resizing. +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +const int dense_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_EMPTY_PCT + = static_cast<int>(0.4 * + dense_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT); + +_END_GOOGLE_NAMESPACE_ + +#endif /* _DENSEHASHTABLE_H_ */ diff --git a/contrib/libs/sparsehash/src/sparsehash/internal/hashtable-common.h b/contrib/libs/sparsehash/src/sparsehash/internal/hashtable-common.h index b22e853830..9e49ec890f 100644 --- a/contrib/libs/sparsehash/src/sparsehash/internal/hashtable-common.h +++ b/contrib/libs/sparsehash/src/sparsehash/internal/hashtable-common.h @@ -1,381 +1,381 @@ -// Copyright (c) 2010, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// -// Provides classes shared by both sparse and dense hashtable. -// -// sh_hashtable_settings has parameters for growing and shrinking -// a hashtable. It also packages zero-size functor (ie. hasher). -// -// Other functions and classes provide common code for serializing -// and deserializing hashtables to a stream (such as a FILE*). - -#ifndef UTIL_GTL_HASHTABLE_COMMON_H_ -#define UTIL_GTL_HASHTABLE_COMMON_H_ - -#include <sparsehash/internal/sparseconfig.h> -#include <assert.h> -#include <stdio.h> -#include <stddef.h> // for size_t -#include <iosfwd> -#include <stdexcept> // For length_error - -_START_GOOGLE_NAMESPACE_ - -template <bool> struct SparsehashCompileAssert { }; -#define SPARSEHASH_COMPILE_ASSERT(expr, msg) \ +// Copyright (c) 2010, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// +// Provides classes shared by both sparse and dense hashtable. +// +// sh_hashtable_settings has parameters for growing and shrinking +// a hashtable. It also packages zero-size functor (ie. hasher). +// +// Other functions and classes provide common code for serializing +// and deserializing hashtables to a stream (such as a FILE*). + +#ifndef UTIL_GTL_HASHTABLE_COMMON_H_ +#define UTIL_GTL_HASHTABLE_COMMON_H_ + +#include <sparsehash/internal/sparseconfig.h> +#include <assert.h> +#include <stdio.h> +#include <stddef.h> // for size_t +#include <iosfwd> +#include <stdexcept> // For length_error + +_START_GOOGLE_NAMESPACE_ + +template <bool> struct SparsehashCompileAssert { }; +#define SPARSEHASH_COMPILE_ASSERT(expr, msg) \ [[maybe_unused]] typedef SparsehashCompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1] - -namespace sparsehash_internal { - -// Adaptor methods for reading/writing data from an INPUT or OUPTUT -// variable passed to serialize() or unserialize(). For now we -// have implemented INPUT/OUTPUT for FILE*, istream*/ostream* (note -// they are pointers, unlike typical use), or else a pointer to -// something that supports a Read()/Write() method. -// -// For technical reasons, we implement read_data/write_data in two -// stages. The actual work is done in *_data_internal, which takes -// the stream argument twice: once as a template type, and once with -// normal type information. (We only use the second version.) We do -// this because of how C++ picks what function overload to use. If we -// implemented this the naive way: -// bool read_data(istream* is, const void* data, size_t length); -// template<typename T> read_data(T* fp, const void* data, size_t length); -// C++ would prefer the second version for every stream type except -// istream. However, we want C++ to prefer the first version for -// streams that are *subclasses* of istream, such as istringstream. -// This is not possible given the way template types are resolved. So -// we split the stream argument in two, one of which is templated and -// one of which is not. The specialized functions (like the istream -// version above) ignore the template arg and use the second, 'type' -// arg, getting subclass matching as normal. The 'catch-all' -// functions (the second version above) use the template arg to deduce -// the type, and use a second, void* arg to achieve the desired -// 'catch-all' semantics. - -// ----- low-level I/O for FILE* ---- - -template<typename Ignored> -inline bool read_data_internal(Ignored*, FILE* fp, - void* data, size_t length) { - return fread(data, length, 1, fp) == 1; -} - -template<typename Ignored> -inline bool write_data_internal(Ignored*, FILE* fp, - const void* data, size_t length) { - return fwrite(data, length, 1, fp) == 1; -} - -// ----- low-level I/O for iostream ---- - -// We want the caller to be responsible for #including <iostream>, not -// us, because iostream is a big header! According to the standard, -// it's only legal to delay the instantiation the way we want to if -// the istream/ostream is a template type. So we jump through hoops. -template<typename ISTREAM> -inline bool read_data_internal_for_istream(ISTREAM* fp, - void* data, size_t length) { - return fp->read(reinterpret_cast<char*>(data), length).good(); -} -template<typename Ignored> -inline bool read_data_internal(Ignored*, std::istream* fp, - void* data, size_t length) { - return read_data_internal_for_istream(fp, data, length); -} - -template<typename OSTREAM> -inline bool write_data_internal_for_ostream(OSTREAM* fp, - const void* data, size_t length) { - return fp->write(reinterpret_cast<const char*>(data), length).good(); -} -template<typename Ignored> -inline bool write_data_internal(Ignored*, std::ostream* fp, - const void* data, size_t length) { - return write_data_internal_for_ostream(fp, data, length); -} - -// ----- low-level I/O for custom streams ---- - -// The INPUT type needs to support a Read() method that takes a -// buffer and a length and returns the number of bytes read. -template <typename INPUT> -inline bool read_data_internal(INPUT* fp, void*, - void* data, size_t length) { - return static_cast<size_t>(fp->Read(data, length)) == length; -} - -// The OUTPUT type needs to support a Write() operation that takes -// a buffer and a length and returns the number of bytes written. -template <typename OUTPUT> -inline bool write_data_internal(OUTPUT* fp, void*, - const void* data, size_t length) { - return static_cast<size_t>(fp->Write(data, length)) == length; -} - -// ----- low-level I/O: the public API ---- - -template <typename INPUT> -inline bool read_data(INPUT* fp, void* data, size_t length) { - return read_data_internal(fp, fp, data, length); -} - -template <typename OUTPUT> -inline bool write_data(OUTPUT* fp, const void* data, size_t length) { - return write_data_internal(fp, fp, data, length); -} - -// Uses read_data() and write_data() to read/write an integer. -// length is the number of bytes to read/write (which may differ -// from sizeof(IntType), allowing us to save on a 32-bit system -// and load on a 64-bit system). Excess bytes are taken to be 0. -// INPUT and OUTPUT must match legal inputs to read/write_data (above). -template <typename INPUT, typename IntType> -bool read_bigendian_number(INPUT* fp, IntType* value, size_t length) { - *value = 0; - unsigned char byte; - // We require IntType to be unsigned or else the shifting gets all screwy. - SPARSEHASH_COMPILE_ASSERT(static_cast<IntType>(-1) > static_cast<IntType>(0), - serializing_int_requires_an_unsigned_type); - for (size_t i = 0; i < length; ++i) { - if (!read_data(fp, &byte, sizeof(byte))) return false; - *value |= static_cast<IntType>(byte) << ((length - 1 - i) * 8); - } - return true; -} - -template <typename OUTPUT, typename IntType> -bool write_bigendian_number(OUTPUT* fp, IntType value, size_t length) { - unsigned char byte; - // We require IntType to be unsigned or else the shifting gets all screwy. - SPARSEHASH_COMPILE_ASSERT(static_cast<IntType>(-1) > static_cast<IntType>(0), - serializing_int_requires_an_unsigned_type); - for (size_t i = 0; i < length; ++i) { - byte = (sizeof(value) <= length-1 - i) - ? 0 : static_cast<unsigned char>((value >> ((length-1 - i) * 8)) & 255); - if (!write_data(fp, &byte, sizeof(byte))) return false; - } - return true; -} - -// If your keys and values are simple enough, you can pass this -// serializer to serialize()/unserialize(). "Simple enough" means -// value_type is a POD type that contains no pointers. Note, -// however, we don't try to normalize endianness. -// This is the type used for NopointerSerializer. -template <typename value_type> struct pod_serializer { - template <typename INPUT> - bool operator()(INPUT* fp, value_type* value) const { - return read_data(fp, value, sizeof(*value)); - } - - template <typename OUTPUT> - bool operator()(OUTPUT* fp, const value_type& value) const { - return write_data(fp, &value, sizeof(value)); - } -}; - - -// Settings contains parameters for growing and shrinking the table. -// It also packages zero-size functor (ie. hasher). -// -// It does some munging of the hash value in cases where we think -// (fear) the original hash function might not be very good. In -// particular, the default hash of pointers is the identity hash, -// so probably all the low bits are 0. We identify when we think -// we're hashing a pointer, and chop off the low bits. Note this -// isn't perfect: even when the key is a pointer, we can't tell -// for sure that the hash is the identity hash. If it's not, this -// is needless work (and possibly, though not likely, harmful). - -template<typename Key, typename HashFunc, - typename SizeType, int HT_MIN_BUCKETS> -class sh_hashtable_settings : public HashFunc { - public: - typedef Key key_type; - typedef HashFunc hasher; - typedef SizeType size_type; - - public: - sh_hashtable_settings(const hasher& hf, - const float ht_occupancy_flt, - const float ht_empty_flt) - : hasher(hf), - enlarge_threshold_(0), - shrink_threshold_(0), - consider_shrink_(false), - use_empty_(false), - use_deleted_(false), - num_ht_copies_(0) { - set_enlarge_factor(ht_occupancy_flt); - set_shrink_factor(ht_empty_flt); - } - - size_type hash(const key_type& v) const { - // We munge the hash value when we don't trust hasher::operator(). - return hash_munger<Key>::MungedHash(hasher::operator()(v)); - } - - float enlarge_factor() const { - return enlarge_factor_; - } - void set_enlarge_factor(float f) { - enlarge_factor_ = f; - } - float shrink_factor() const { - return shrink_factor_; - } - void set_shrink_factor(float f) { - shrink_factor_ = f; - } - - size_type enlarge_threshold() const { - return enlarge_threshold_; - } - void set_enlarge_threshold(size_type t) { - enlarge_threshold_ = t; - } - size_type shrink_threshold() const { - return shrink_threshold_; - } - void set_shrink_threshold(size_type t) { - shrink_threshold_ = t; - } - - size_type enlarge_size(size_type x) const { - return static_cast<size_type>(x * enlarge_factor_); - } - size_type shrink_size(size_type x) const { - return static_cast<size_type>(x * shrink_factor_); - } - - bool consider_shrink() const { - return consider_shrink_; - } - void set_consider_shrink(bool t) { - consider_shrink_ = t; - } - - bool use_empty() const { - return use_empty_; - } - void set_use_empty(bool t) { - use_empty_ = t; - } - - bool use_deleted() const { - return use_deleted_; - } - void set_use_deleted(bool t) { - use_deleted_ = t; - } - - size_type num_ht_copies() const { - return static_cast<size_type>(num_ht_copies_); - } - void inc_num_ht_copies() { - ++num_ht_copies_; - } - - // Reset the enlarge and shrink thresholds - void reset_thresholds(size_type num_buckets) { - set_enlarge_threshold(enlarge_size(num_buckets)); - set_shrink_threshold(shrink_size(num_buckets)); - // whatever caused us to reset already considered - set_consider_shrink(false); - } - - // Caller is resposible for calling reset_threshold right after - // set_resizing_parameters. - void set_resizing_parameters(float shrink, float grow) { - assert(shrink >= 0.0); - assert(grow <= 1.0); - if (shrink > grow/2.0f) - shrink = grow / 2.0f; // otherwise we thrash hashtable size - set_shrink_factor(shrink); - set_enlarge_factor(grow); - } - - // This is the smallest size a hashtable can be without being too crowded - // If you like, you can give a min #buckets as well as a min #elts - size_type min_buckets(size_type num_elts, size_type min_buckets_wanted) { - float enlarge = enlarge_factor(); - size_type sz = HT_MIN_BUCKETS; // min buckets allowed - while ( sz < min_buckets_wanted || - num_elts >= static_cast<size_type>(sz * enlarge) ) { - // This just prevents overflowing size_type, since sz can exceed - // max_size() here. - if (static_cast<size_type>(sz * 2) < sz) { - throw std::length_error("resize overflow"); // protect against overflow - } - sz *= 2; - } - return sz; - } - - private: - template<class HashKey> class hash_munger { - public: - static size_t MungedHash(size_t hash) { - return hash; - } - }; - // This matches when the hashtable key is a pointer. - template<class HashKey> class hash_munger<HashKey*> { - public: - static size_t MungedHash(size_t hash) { - // TODO(csilvers): consider rotating instead: - // static const int shift = (sizeof(void *) == 4) ? 2 : 3; - // return (hash << (sizeof(hash) * 8) - shift)) | (hash >> shift); - // This matters if we ever change sparse/dense_hash_* to compare - // hashes before comparing actual values. It's speedy on x86. - return hash / sizeof(void*); // get rid of known-0 bits - } - }; - - size_type enlarge_threshold_; // table.size() * enlarge_factor - size_type shrink_threshold_; // table.size() * shrink_factor - float enlarge_factor_; // how full before resize - float shrink_factor_; // how empty before resize - // consider_shrink=true if we should try to shrink before next insert - bool consider_shrink_; - bool use_empty_; // used only by densehashtable, not sparsehashtable - bool use_deleted_; // false until delkey has been set - // num_ht_copies is a counter incremented every Copy/Move - unsigned int num_ht_copies_; -}; - -} // namespace sparsehash_internal - -#undef SPARSEHASH_COMPILE_ASSERT -_END_GOOGLE_NAMESPACE_ - -#endif // UTIL_GTL_HASHTABLE_COMMON_H_ + +namespace sparsehash_internal { + +// Adaptor methods for reading/writing data from an INPUT or OUPTUT +// variable passed to serialize() or unserialize(). For now we +// have implemented INPUT/OUTPUT for FILE*, istream*/ostream* (note +// they are pointers, unlike typical use), or else a pointer to +// something that supports a Read()/Write() method. +// +// For technical reasons, we implement read_data/write_data in two +// stages. The actual work is done in *_data_internal, which takes +// the stream argument twice: once as a template type, and once with +// normal type information. (We only use the second version.) We do +// this because of how C++ picks what function overload to use. If we +// implemented this the naive way: +// bool read_data(istream* is, const void* data, size_t length); +// template<typename T> read_data(T* fp, const void* data, size_t length); +// C++ would prefer the second version for every stream type except +// istream. However, we want C++ to prefer the first version for +// streams that are *subclasses* of istream, such as istringstream. +// This is not possible given the way template types are resolved. So +// we split the stream argument in two, one of which is templated and +// one of which is not. The specialized functions (like the istream +// version above) ignore the template arg and use the second, 'type' +// arg, getting subclass matching as normal. The 'catch-all' +// functions (the second version above) use the template arg to deduce +// the type, and use a second, void* arg to achieve the desired +// 'catch-all' semantics. + +// ----- low-level I/O for FILE* ---- + +template<typename Ignored> +inline bool read_data_internal(Ignored*, FILE* fp, + void* data, size_t length) { + return fread(data, length, 1, fp) == 1; +} + +template<typename Ignored> +inline bool write_data_internal(Ignored*, FILE* fp, + const void* data, size_t length) { + return fwrite(data, length, 1, fp) == 1; +} + +// ----- low-level I/O for iostream ---- + +// We want the caller to be responsible for #including <iostream>, not +// us, because iostream is a big header! According to the standard, +// it's only legal to delay the instantiation the way we want to if +// the istream/ostream is a template type. So we jump through hoops. +template<typename ISTREAM> +inline bool read_data_internal_for_istream(ISTREAM* fp, + void* data, size_t length) { + return fp->read(reinterpret_cast<char*>(data), length).good(); +} +template<typename Ignored> +inline bool read_data_internal(Ignored*, std::istream* fp, + void* data, size_t length) { + return read_data_internal_for_istream(fp, data, length); +} + +template<typename OSTREAM> +inline bool write_data_internal_for_ostream(OSTREAM* fp, + const void* data, size_t length) { + return fp->write(reinterpret_cast<const char*>(data), length).good(); +} +template<typename Ignored> +inline bool write_data_internal(Ignored*, std::ostream* fp, + const void* data, size_t length) { + return write_data_internal_for_ostream(fp, data, length); +} + +// ----- low-level I/O for custom streams ---- + +// The INPUT type needs to support a Read() method that takes a +// buffer and a length and returns the number of bytes read. +template <typename INPUT> +inline bool read_data_internal(INPUT* fp, void*, + void* data, size_t length) { + return static_cast<size_t>(fp->Read(data, length)) == length; +} + +// The OUTPUT type needs to support a Write() operation that takes +// a buffer and a length and returns the number of bytes written. +template <typename OUTPUT> +inline bool write_data_internal(OUTPUT* fp, void*, + const void* data, size_t length) { + return static_cast<size_t>(fp->Write(data, length)) == length; +} + +// ----- low-level I/O: the public API ---- + +template <typename INPUT> +inline bool read_data(INPUT* fp, void* data, size_t length) { + return read_data_internal(fp, fp, data, length); +} + +template <typename OUTPUT> +inline bool write_data(OUTPUT* fp, const void* data, size_t length) { + return write_data_internal(fp, fp, data, length); +} + +// Uses read_data() and write_data() to read/write an integer. +// length is the number of bytes to read/write (which may differ +// from sizeof(IntType), allowing us to save on a 32-bit system +// and load on a 64-bit system). Excess bytes are taken to be 0. +// INPUT and OUTPUT must match legal inputs to read/write_data (above). +template <typename INPUT, typename IntType> +bool read_bigendian_number(INPUT* fp, IntType* value, size_t length) { + *value = 0; + unsigned char byte; + // We require IntType to be unsigned or else the shifting gets all screwy. + SPARSEHASH_COMPILE_ASSERT(static_cast<IntType>(-1) > static_cast<IntType>(0), + serializing_int_requires_an_unsigned_type); + for (size_t i = 0; i < length; ++i) { + if (!read_data(fp, &byte, sizeof(byte))) return false; + *value |= static_cast<IntType>(byte) << ((length - 1 - i) * 8); + } + return true; +} + +template <typename OUTPUT, typename IntType> +bool write_bigendian_number(OUTPUT* fp, IntType value, size_t length) { + unsigned char byte; + // We require IntType to be unsigned or else the shifting gets all screwy. + SPARSEHASH_COMPILE_ASSERT(static_cast<IntType>(-1) > static_cast<IntType>(0), + serializing_int_requires_an_unsigned_type); + for (size_t i = 0; i < length; ++i) { + byte = (sizeof(value) <= length-1 - i) + ? 0 : static_cast<unsigned char>((value >> ((length-1 - i) * 8)) & 255); + if (!write_data(fp, &byte, sizeof(byte))) return false; + } + return true; +} + +// If your keys and values are simple enough, you can pass this +// serializer to serialize()/unserialize(). "Simple enough" means +// value_type is a POD type that contains no pointers. Note, +// however, we don't try to normalize endianness. +// This is the type used for NopointerSerializer. +template <typename value_type> struct pod_serializer { + template <typename INPUT> + bool operator()(INPUT* fp, value_type* value) const { + return read_data(fp, value, sizeof(*value)); + } + + template <typename OUTPUT> + bool operator()(OUTPUT* fp, const value_type& value) const { + return write_data(fp, &value, sizeof(value)); + } +}; + + +// Settings contains parameters for growing and shrinking the table. +// It also packages zero-size functor (ie. hasher). +// +// It does some munging of the hash value in cases where we think +// (fear) the original hash function might not be very good. In +// particular, the default hash of pointers is the identity hash, +// so probably all the low bits are 0. We identify when we think +// we're hashing a pointer, and chop off the low bits. Note this +// isn't perfect: even when the key is a pointer, we can't tell +// for sure that the hash is the identity hash. If it's not, this +// is needless work (and possibly, though not likely, harmful). + +template<typename Key, typename HashFunc, + typename SizeType, int HT_MIN_BUCKETS> +class sh_hashtable_settings : public HashFunc { + public: + typedef Key key_type; + typedef HashFunc hasher; + typedef SizeType size_type; + + public: + sh_hashtable_settings(const hasher& hf, + const float ht_occupancy_flt, + const float ht_empty_flt) + : hasher(hf), + enlarge_threshold_(0), + shrink_threshold_(0), + consider_shrink_(false), + use_empty_(false), + use_deleted_(false), + num_ht_copies_(0) { + set_enlarge_factor(ht_occupancy_flt); + set_shrink_factor(ht_empty_flt); + } + + size_type hash(const key_type& v) const { + // We munge the hash value when we don't trust hasher::operator(). + return hash_munger<Key>::MungedHash(hasher::operator()(v)); + } + + float enlarge_factor() const { + return enlarge_factor_; + } + void set_enlarge_factor(float f) { + enlarge_factor_ = f; + } + float shrink_factor() const { + return shrink_factor_; + } + void set_shrink_factor(float f) { + shrink_factor_ = f; + } + + size_type enlarge_threshold() const { + return enlarge_threshold_; + } + void set_enlarge_threshold(size_type t) { + enlarge_threshold_ = t; + } + size_type shrink_threshold() const { + return shrink_threshold_; + } + void set_shrink_threshold(size_type t) { + shrink_threshold_ = t; + } + + size_type enlarge_size(size_type x) const { + return static_cast<size_type>(x * enlarge_factor_); + } + size_type shrink_size(size_type x) const { + return static_cast<size_type>(x * shrink_factor_); + } + + bool consider_shrink() const { + return consider_shrink_; + } + void set_consider_shrink(bool t) { + consider_shrink_ = t; + } + + bool use_empty() const { + return use_empty_; + } + void set_use_empty(bool t) { + use_empty_ = t; + } + + bool use_deleted() const { + return use_deleted_; + } + void set_use_deleted(bool t) { + use_deleted_ = t; + } + + size_type num_ht_copies() const { + return static_cast<size_type>(num_ht_copies_); + } + void inc_num_ht_copies() { + ++num_ht_copies_; + } + + // Reset the enlarge and shrink thresholds + void reset_thresholds(size_type num_buckets) { + set_enlarge_threshold(enlarge_size(num_buckets)); + set_shrink_threshold(shrink_size(num_buckets)); + // whatever caused us to reset already considered + set_consider_shrink(false); + } + + // Caller is resposible for calling reset_threshold right after + // set_resizing_parameters. + void set_resizing_parameters(float shrink, float grow) { + assert(shrink >= 0.0); + assert(grow <= 1.0); + if (shrink > grow/2.0f) + shrink = grow / 2.0f; // otherwise we thrash hashtable size + set_shrink_factor(shrink); + set_enlarge_factor(grow); + } + + // This is the smallest size a hashtable can be without being too crowded + // If you like, you can give a min #buckets as well as a min #elts + size_type min_buckets(size_type num_elts, size_type min_buckets_wanted) { + float enlarge = enlarge_factor(); + size_type sz = HT_MIN_BUCKETS; // min buckets allowed + while ( sz < min_buckets_wanted || + num_elts >= static_cast<size_type>(sz * enlarge) ) { + // This just prevents overflowing size_type, since sz can exceed + // max_size() here. + if (static_cast<size_type>(sz * 2) < sz) { + throw std::length_error("resize overflow"); // protect against overflow + } + sz *= 2; + } + return sz; + } + + private: + template<class HashKey> class hash_munger { + public: + static size_t MungedHash(size_t hash) { + return hash; + } + }; + // This matches when the hashtable key is a pointer. + template<class HashKey> class hash_munger<HashKey*> { + public: + static size_t MungedHash(size_t hash) { + // TODO(csilvers): consider rotating instead: + // static const int shift = (sizeof(void *) == 4) ? 2 : 3; + // return (hash << (sizeof(hash) * 8) - shift)) | (hash >> shift); + // This matters if we ever change sparse/dense_hash_* to compare + // hashes before comparing actual values. It's speedy on x86. + return hash / sizeof(void*); // get rid of known-0 bits + } + }; + + size_type enlarge_threshold_; // table.size() * enlarge_factor + size_type shrink_threshold_; // table.size() * shrink_factor + float enlarge_factor_; // how full before resize + float shrink_factor_; // how empty before resize + // consider_shrink=true if we should try to shrink before next insert + bool consider_shrink_; + bool use_empty_; // used only by densehashtable, not sparsehashtable + bool use_deleted_; // false until delkey has been set + // num_ht_copies is a counter incremented every Copy/Move + unsigned int num_ht_copies_; +}; + +} // namespace sparsehash_internal + +#undef SPARSEHASH_COMPILE_ASSERT +_END_GOOGLE_NAMESPACE_ + +#endif // UTIL_GTL_HASHTABLE_COMMON_H_ diff --git a/contrib/libs/sparsehash/src/sparsehash/internal/libc_allocator_with_realloc.h b/contrib/libs/sparsehash/src/sparsehash/internal/libc_allocator_with_realloc.h index 9429b84b7a..769329fa6c 100644 --- a/contrib/libs/sparsehash/src/sparsehash/internal/libc_allocator_with_realloc.h +++ b/contrib/libs/sparsehash/src/sparsehash/internal/libc_allocator_with_realloc.h @@ -1,122 +1,122 @@ -// Copyright (c) 2010, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- - -#ifndef UTIL_GTL_LIBC_ALLOCATOR_WITH_REALLOC_H_ -#define UTIL_GTL_LIBC_ALLOCATOR_WITH_REALLOC_H_ - -#include <sparsehash/internal/sparseconfig.h> -#include <stdlib.h> // for malloc/realloc/free -#include <stddef.h> // for ptrdiff_t -#include <new> // for placement new - -_START_GOOGLE_NAMESPACE_ - -template<class T> -class libc_allocator_with_realloc { - public: - typedef T value_type; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - - typedef T* pointer; - typedef const T* const_pointer; - typedef T& reference; - typedef const T& const_reference; - - libc_allocator_with_realloc() {} - libc_allocator_with_realloc(const libc_allocator_with_realloc&) {} - ~libc_allocator_with_realloc() {} - - pointer address(reference r) const { return &r; } - const_pointer address(const_reference r) const { return &r; } - - pointer allocate(size_type n, const_pointer = 0) { - return static_cast<pointer>(malloc(n * sizeof(value_type))); - } - void deallocate(pointer p, size_type) { - free(p); - } - pointer reallocate(pointer p, size_type n) { +// Copyright (c) 2010, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- + +#ifndef UTIL_GTL_LIBC_ALLOCATOR_WITH_REALLOC_H_ +#define UTIL_GTL_LIBC_ALLOCATOR_WITH_REALLOC_H_ + +#include <sparsehash/internal/sparseconfig.h> +#include <stdlib.h> // for malloc/realloc/free +#include <stddef.h> // for ptrdiff_t +#include <new> // for placement new + +_START_GOOGLE_NAMESPACE_ + +template<class T> +class libc_allocator_with_realloc { + public: + typedef T value_type; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + + typedef T* pointer; + typedef const T* const_pointer; + typedef T& reference; + typedef const T& const_reference; + + libc_allocator_with_realloc() {} + libc_allocator_with_realloc(const libc_allocator_with_realloc&) {} + ~libc_allocator_with_realloc() {} + + pointer address(reference r) const { return &r; } + const_pointer address(const_reference r) const { return &r; } + + pointer allocate(size_type n, const_pointer = 0) { + return static_cast<pointer>(malloc(n * sizeof(value_type))); + } + void deallocate(pointer p, size_type) { + free(p); + } + pointer reallocate(pointer p, size_type n) { // p points to a storage array whose objects have already been destroyed // cast to void* to prevent compiler warnings about calling realloc() on // an object which cannot be relocated in memory return static_cast<pointer>(realloc(static_cast<void*>(p), n * sizeof(value_type))); - } - - size_type max_size() const { - return static_cast<size_type>(-1) / sizeof(value_type); - } - - void construct(pointer p, const value_type& val) { - new(p) value_type(val); - } - void destroy(pointer p) { p->~value_type(); } - - template <class U> - libc_allocator_with_realloc(const libc_allocator_with_realloc<U>&) {} - - template<class U> - struct rebind { - typedef libc_allocator_with_realloc<U> other; - }; -}; - -// libc_allocator_with_realloc<void> specialization. -template<> -class libc_allocator_with_realloc<void> { - public: - typedef void value_type; - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef void* pointer; - typedef const void* const_pointer; - - template<class U> - struct rebind { - typedef libc_allocator_with_realloc<U> other; - }; -}; - -template<class T> -inline bool operator==(const libc_allocator_with_realloc<T>&, - const libc_allocator_with_realloc<T>&) { - return true; -} - -template<class T> -inline bool operator!=(const libc_allocator_with_realloc<T>&, - const libc_allocator_with_realloc<T>&) { - return false; -} - -_END_GOOGLE_NAMESPACE_ - -#endif // UTIL_GTL_LIBC_ALLOCATOR_WITH_REALLOC_H_ + } + + size_type max_size() const { + return static_cast<size_type>(-1) / sizeof(value_type); + } + + void construct(pointer p, const value_type& val) { + new(p) value_type(val); + } + void destroy(pointer p) { p->~value_type(); } + + template <class U> + libc_allocator_with_realloc(const libc_allocator_with_realloc<U>&) {} + + template<class U> + struct rebind { + typedef libc_allocator_with_realloc<U> other; + }; +}; + +// libc_allocator_with_realloc<void> specialization. +template<> +class libc_allocator_with_realloc<void> { + public: + typedef void value_type; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef void* pointer; + typedef const void* const_pointer; + + template<class U> + struct rebind { + typedef libc_allocator_with_realloc<U> other; + }; +}; + +template<class T> +inline bool operator==(const libc_allocator_with_realloc<T>&, + const libc_allocator_with_realloc<T>&) { + return true; +} + +template<class T> +inline bool operator!=(const libc_allocator_with_realloc<T>&, + const libc_allocator_with_realloc<T>&) { + return false; +} + +_END_GOOGLE_NAMESPACE_ + +#endif // UTIL_GTL_LIBC_ALLOCATOR_WITH_REALLOC_H_ diff --git a/contrib/libs/sparsehash/src/sparsehash/internal/sparseconfig.h b/contrib/libs/sparsehash/src/sparsehash/internal/sparseconfig.h index 42d3dab03d..bb8c9a9428 100644 --- a/contrib/libs/sparsehash/src/sparsehash/internal/sparseconfig.h +++ b/contrib/libs/sparsehash/src/sparsehash/internal/sparseconfig.h @@ -1,48 +1,48 @@ -/* - * NOTE: This file is for internal use only. - * Do not use these #defines in your own program! - */ - -/* Namespace for Google classes */ +/* + * NOTE: This file is for internal use only. + * Do not use these #defines in your own program! + */ + +/* Namespace for Google classes */ #define GOOGLE_NAMESPACE ::google - + /* the location of the header defining hash functions */ #ifndef HASH_FUN_H -#define HASH_FUN_H <util/generic/hash.h> +#define HASH_FUN_H <util/generic/hash.h> #endif - -/* the namespace of the hash<> function */ -#define HASH_NAMESPACE - -/* Define to 1 if you have the <inttypes.h> header file. */ -#define HAVE_INTTYPES_H 1 - -/* Define to 1 if the system has the type `long long'. */ -#define HAVE_LONG_LONG 1 - -/* Define to 1 if you have the `memcpy' function. */ -#define HAVE_MEMCPY 1 - -/* Define to 1 if you have the <stdint.h> header file. */ -#define HAVE_STDINT_H 1 - -/* Define to 1 if you have the <sys/types.h> header file. */ -#define HAVE_SYS_TYPES_H 1 - -/* Define to 1 if the system has the type `uint16_t'. */ -#define HAVE_UINT16_T 1 - -/* Define to 1 if the system has the type `u_int16_t'. */ -#define HAVE_U_INT16_T 1 - -/* Define to 1 if the system has the type `__uint16'. */ -/* #undef HAVE___UINT16 */ - -/* The system-provided hash function including the namespace. */ + +/* the namespace of the hash<> function */ +#define HASH_NAMESPACE + +/* Define to 1 if you have the <inttypes.h> header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if the system has the type `long long'. */ +#define HAVE_LONG_LONG 1 + +/* Define to 1 if you have the `memcpy' function. */ +#define HAVE_MEMCPY 1 + +/* Define to 1 if you have the <stdint.h> header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the <sys/types.h> header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if the system has the type `uint16_t'. */ +#define HAVE_UINT16_T 1 + +/* Define to 1 if the system has the type `u_int16_t'. */ +#define HAVE_U_INT16_T 1 + +/* Define to 1 if the system has the type `__uint16'. */ +/* #undef HAVE___UINT16 */ + +/* The system-provided hash function including the namespace. */ #define SPARSEHASH_HASH HASH_NAMESPACE::THash - -/* Stops putting the code inside the Google namespace */ + +/* Stops putting the code inside the Google namespace */ #define _END_GOOGLE_NAMESPACE_ } - -/* Puts following code inside the Google namespace */ + +/* Puts following code inside the Google namespace */ #define _START_GOOGLE_NAMESPACE_ namespace google { diff --git a/contrib/libs/sparsehash/src/sparsehash/internal/sparsehashtable.h b/contrib/libs/sparsehash/src/sparsehash/internal/sparsehashtable.h index 50eb01b767..f54ea51e9a 100644 --- a/contrib/libs/sparsehash/src/sparsehash/internal/sparsehashtable.h +++ b/contrib/libs/sparsehash/src/sparsehash/internal/sparsehashtable.h @@ -1,1247 +1,1247 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// -// A sparse hashtable is a particular implementation of -// a hashtable: one that is meant to minimize memory use. -// It does this by using a *sparse table* (cf sparsetable.h), -// which uses between 1 and 2 bits to store empty buckets -// (we may need another bit for hashtables that support deletion). -// -// When empty buckets are so cheap, an appealing hashtable -// implementation is internal probing, in which the hashtable -// is a single table, and collisions are resolved by trying -// to insert again in another bucket. The most cache-efficient -// internal probing schemes are linear probing (which suffers, -// alas, from clumping) and quadratic probing, which is what -// we implement by default. -// -// Deleted buckets are a bit of a pain. We have to somehow mark -// deleted buckets (the probing must distinguish them from empty -// buckets). The most principled way is to have another bitmap, -// but that's annoying and takes up space. Instead we let the -// user specify an "impossible" key. We set deleted buckets -// to have the impossible key. -// -// Note it is possible to change the value of the delete key -// on the fly; you can even remove it, though after that point -// the hashtable is insert_only until you set it again. -// -// You probably shouldn't use this code directly. Use -// sparse_hash_map<> or sparse_hash_set<> instead. -// -// You can modify the following, below: -// HT_OCCUPANCY_PCT -- how full before we double size -// HT_EMPTY_PCT -- how empty before we halve size -// HT_MIN_BUCKETS -- smallest bucket size -// HT_DEFAULT_STARTING_BUCKETS -- default bucket size at construct-time -// -// You can also change enlarge_factor (which defaults to -// HT_OCCUPANCY_PCT), and shrink_factor (which defaults to -// HT_EMPTY_PCT) with set_resizing_parameters(). -// -// How to decide what values to use? -// shrink_factor's default of .4 * OCCUPANCY_PCT, is probably good. -// HT_MIN_BUCKETS is probably unnecessary since you can specify -// (indirectly) the starting number of buckets at construct-time. -// For enlarge_factor, you can use this chart to try to trade-off -// expected lookup time to the space taken up. By default, this -// code uses quadratic probing, though you can change it to linear -// via _JUMP below if you really want to. -// -// From http://www.augustana.ca/~mohrj/courses/1999.fall/csc210/lecture_notes/hashing.html -// NUMBER OF PROBES / LOOKUP Successful Unsuccessful -// Quadratic collision resolution 1 - ln(1-L) - L/2 1/(1-L) - L - ln(1-L) -// Linear collision resolution [1+1/(1-L)]/2 [1+1/(1-L)2]/2 -// -// -- enlarge_factor -- 0.10 0.50 0.60 0.75 0.80 0.90 0.99 -// QUADRATIC COLLISION RES. -// probes/successful lookup 1.05 1.44 1.62 2.01 2.21 2.85 5.11 -// probes/unsuccessful lookup 1.11 2.19 2.82 4.64 5.81 11.4 103.6 -// LINEAR COLLISION RES. -// probes/successful lookup 1.06 1.5 1.75 2.5 3.0 5.5 50.5 -// probes/unsuccessful lookup 1.12 2.5 3.6 8.5 13.0 50.0 5000.0 -// -// The value type is required to be copy constructible and default -// constructible, but it need not be (and commonly isn't) assignable. - -#ifndef _SPARSEHASHTABLE_H_ -#define _SPARSEHASHTABLE_H_ - -#include <sparsehash/internal/sparseconfig.h> -#include <assert.h> -#include <algorithm> // For swap(), eg -#include <iterator> // for iterator tags -#include <limits> // for numeric_limits -#include <utility> // for pair -#include <sparsehash/type_traits.h> // for remove_const -#include <sparsehash/internal/hashtable-common.h> -#include <sparsehash/sparsetable> // IWYU pragma: export -#include <stdexcept> // For length_error - -_START_GOOGLE_NAMESPACE_ - -namespace base { // just to make google->opensource transition easier -using GOOGLE_NAMESPACE::remove_const; -} - -#ifndef SPARSEHASH_STAT_UPDATE -#define SPARSEHASH_STAT_UPDATE(x) ((void) 0) -#endif - -// The probing method -// Linear probing -// #define JUMP_(key, num_probes) ( 1 ) -// Quadratic probing -#define JUMP_(key, num_probes) ( num_probes ) - -// The smaller this is, the faster lookup is (because the group bitmap is -// smaller) and the faster insert is, because there's less to move. -// On the other hand, there are more groups. Since group::size_type is -// a short, this number should be of the form 32*x + 16 to avoid waste. -static const u_int16_t DEFAULT_GROUP_SIZE = 48; // fits in 1.5 words - -// Hashtable class, used to implement the hashed associative containers -// hash_set and hash_map. -// -// Value: what is stored in the table (each bucket is a Value). -// Key: something in a 1-to-1 correspondence to a Value, that can be used -// to search for a Value in the table (find() takes a Key). -// HashFcn: Takes a Key and returns an integer, the more unique the better. -// ExtractKey: given a Value, returns the unique Key associated with it. -// Must inherit from unary_function, or at least have a -// result_type enum indicating the return type of operator(). -// SetKey: given a Value* and a Key, modifies the value such that -// ExtractKey(value) == key. We guarantee this is only called -// with key == deleted_key. -// EqualKey: Given two Keys, says whether they are the same (that is, -// if they are both associated with the same Value). -// Alloc: STL allocator to use to allocate memory. - -template <class Value, class Key, class HashFcn, - class ExtractKey, class SetKey, class EqualKey, class Alloc> -class sparse_hashtable; - -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -struct sparse_hashtable_iterator; - -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -struct sparse_hashtable_const_iterator; - -// As far as iterating, we're basically just a sparsetable -// that skips over deleted elements. -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -struct sparse_hashtable_iterator { - private: - typedef typename A::template rebind<V>::other value_alloc_type; - - public: - typedef sparse_hashtable_iterator<V,K,HF,ExK,SetK,EqK,A> iterator; - typedef sparse_hashtable_const_iterator<V,K,HF,ExK,SetK,EqK,A> const_iterator; +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// +// A sparse hashtable is a particular implementation of +// a hashtable: one that is meant to minimize memory use. +// It does this by using a *sparse table* (cf sparsetable.h), +// which uses between 1 and 2 bits to store empty buckets +// (we may need another bit for hashtables that support deletion). +// +// When empty buckets are so cheap, an appealing hashtable +// implementation is internal probing, in which the hashtable +// is a single table, and collisions are resolved by trying +// to insert again in another bucket. The most cache-efficient +// internal probing schemes are linear probing (which suffers, +// alas, from clumping) and quadratic probing, which is what +// we implement by default. +// +// Deleted buckets are a bit of a pain. We have to somehow mark +// deleted buckets (the probing must distinguish them from empty +// buckets). The most principled way is to have another bitmap, +// but that's annoying and takes up space. Instead we let the +// user specify an "impossible" key. We set deleted buckets +// to have the impossible key. +// +// Note it is possible to change the value of the delete key +// on the fly; you can even remove it, though after that point +// the hashtable is insert_only until you set it again. +// +// You probably shouldn't use this code directly. Use +// sparse_hash_map<> or sparse_hash_set<> instead. +// +// You can modify the following, below: +// HT_OCCUPANCY_PCT -- how full before we double size +// HT_EMPTY_PCT -- how empty before we halve size +// HT_MIN_BUCKETS -- smallest bucket size +// HT_DEFAULT_STARTING_BUCKETS -- default bucket size at construct-time +// +// You can also change enlarge_factor (which defaults to +// HT_OCCUPANCY_PCT), and shrink_factor (which defaults to +// HT_EMPTY_PCT) with set_resizing_parameters(). +// +// How to decide what values to use? +// shrink_factor's default of .4 * OCCUPANCY_PCT, is probably good. +// HT_MIN_BUCKETS is probably unnecessary since you can specify +// (indirectly) the starting number of buckets at construct-time. +// For enlarge_factor, you can use this chart to try to trade-off +// expected lookup time to the space taken up. By default, this +// code uses quadratic probing, though you can change it to linear +// via _JUMP below if you really want to. +// +// From http://www.augustana.ca/~mohrj/courses/1999.fall/csc210/lecture_notes/hashing.html +// NUMBER OF PROBES / LOOKUP Successful Unsuccessful +// Quadratic collision resolution 1 - ln(1-L) - L/2 1/(1-L) - L - ln(1-L) +// Linear collision resolution [1+1/(1-L)]/2 [1+1/(1-L)2]/2 +// +// -- enlarge_factor -- 0.10 0.50 0.60 0.75 0.80 0.90 0.99 +// QUADRATIC COLLISION RES. +// probes/successful lookup 1.05 1.44 1.62 2.01 2.21 2.85 5.11 +// probes/unsuccessful lookup 1.11 2.19 2.82 4.64 5.81 11.4 103.6 +// LINEAR COLLISION RES. +// probes/successful lookup 1.06 1.5 1.75 2.5 3.0 5.5 50.5 +// probes/unsuccessful lookup 1.12 2.5 3.6 8.5 13.0 50.0 5000.0 +// +// The value type is required to be copy constructible and default +// constructible, but it need not be (and commonly isn't) assignable. + +#ifndef _SPARSEHASHTABLE_H_ +#define _SPARSEHASHTABLE_H_ + +#include <sparsehash/internal/sparseconfig.h> +#include <assert.h> +#include <algorithm> // For swap(), eg +#include <iterator> // for iterator tags +#include <limits> // for numeric_limits +#include <utility> // for pair +#include <sparsehash/type_traits.h> // for remove_const +#include <sparsehash/internal/hashtable-common.h> +#include <sparsehash/sparsetable> // IWYU pragma: export +#include <stdexcept> // For length_error + +_START_GOOGLE_NAMESPACE_ + +namespace base { // just to make google->opensource transition easier +using GOOGLE_NAMESPACE::remove_const; +} + +#ifndef SPARSEHASH_STAT_UPDATE +#define SPARSEHASH_STAT_UPDATE(x) ((void) 0) +#endif + +// The probing method +// Linear probing +// #define JUMP_(key, num_probes) ( 1 ) +// Quadratic probing +#define JUMP_(key, num_probes) ( num_probes ) + +// The smaller this is, the faster lookup is (because the group bitmap is +// smaller) and the faster insert is, because there's less to move. +// On the other hand, there are more groups. Since group::size_type is +// a short, this number should be of the form 32*x + 16 to avoid waste. +static const u_int16_t DEFAULT_GROUP_SIZE = 48; // fits in 1.5 words + +// Hashtable class, used to implement the hashed associative containers +// hash_set and hash_map. +// +// Value: what is stored in the table (each bucket is a Value). +// Key: something in a 1-to-1 correspondence to a Value, that can be used +// to search for a Value in the table (find() takes a Key). +// HashFcn: Takes a Key and returns an integer, the more unique the better. +// ExtractKey: given a Value, returns the unique Key associated with it. +// Must inherit from unary_function, or at least have a +// result_type enum indicating the return type of operator(). +// SetKey: given a Value* and a Key, modifies the value such that +// ExtractKey(value) == key. We guarantee this is only called +// with key == deleted_key. +// EqualKey: Given two Keys, says whether they are the same (that is, +// if they are both associated with the same Value). +// Alloc: STL allocator to use to allocate memory. + +template <class Value, class Key, class HashFcn, + class ExtractKey, class SetKey, class EqualKey, class Alloc> +class sparse_hashtable; + +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +struct sparse_hashtable_iterator; + +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +struct sparse_hashtable_const_iterator; + +// As far as iterating, we're basically just a sparsetable +// that skips over deleted elements. +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +struct sparse_hashtable_iterator { + private: + typedef typename A::template rebind<V>::other value_alloc_type; + + public: + typedef sparse_hashtable_iterator<V,K,HF,ExK,SetK,EqK,A> iterator; + typedef sparse_hashtable_const_iterator<V,K,HF,ExK,SetK,EqK,A> const_iterator; typedef typename sparsetable<V,DEFAULT_GROUP_SIZE,value_alloc_type>::nonempty_iterator - st_iterator; - - typedef std::forward_iterator_tag iterator_category; // very little defined! - typedef V value_type; - typedef typename value_alloc_type::difference_type difference_type; - typedef typename value_alloc_type::size_type size_type; - typedef typename value_alloc_type::reference reference; - typedef typename value_alloc_type::pointer pointer; - - // "Real" constructor and default constructor - sparse_hashtable_iterator(const sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *h, - st_iterator it, st_iterator it_end) - : ht(h), pos(it), end(it_end) { advance_past_deleted(); } - sparse_hashtable_iterator() { } // not ever used internally - // The default destructor is fine; we don't define one - // The default operator= is fine; we don't define one - - // Happy dereferencer - reference operator*() const { return *pos; } - pointer operator->() const { return &(operator*()); } - - // Arithmetic. The only hard part is making sure that - // we're not on a marked-deleted array element - void advance_past_deleted() { - while ( pos != end && ht->test_deleted(*this) ) - ++pos; - } - iterator& operator++() { - assert(pos != end); ++pos; advance_past_deleted(); return *this; - } - iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } - - // Comparison. - bool operator==(const iterator& it) const { return pos == it.pos; } - bool operator!=(const iterator& it) const { return pos != it.pos; } - - - // The actual data - const sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *ht; - st_iterator pos, end; -}; - -// Now do it all again, but with const-ness! -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -struct sparse_hashtable_const_iterator { - private: - typedef typename A::template rebind<V>::other value_alloc_type; - - public: - typedef sparse_hashtable_iterator<V,K,HF,ExK,SetK,EqK,A> iterator; - typedef sparse_hashtable_const_iterator<V,K,HF,ExK,SetK,EqK,A> const_iterator; + st_iterator; + + typedef std::forward_iterator_tag iterator_category; // very little defined! + typedef V value_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::pointer pointer; + + // "Real" constructor and default constructor + sparse_hashtable_iterator(const sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *h, + st_iterator it, st_iterator it_end) + : ht(h), pos(it), end(it_end) { advance_past_deleted(); } + sparse_hashtable_iterator() { } // not ever used internally + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *pos; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic. The only hard part is making sure that + // we're not on a marked-deleted array element + void advance_past_deleted() { + while ( pos != end && ht->test_deleted(*this) ) + ++pos; + } + iterator& operator++() { + assert(pos != end); ++pos; advance_past_deleted(); return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + + // Comparison. + bool operator==(const iterator& it) const { return pos == it.pos; } + bool operator!=(const iterator& it) const { return pos != it.pos; } + + + // The actual data + const sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *ht; + st_iterator pos, end; +}; + +// Now do it all again, but with const-ness! +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +struct sparse_hashtable_const_iterator { + private: + typedef typename A::template rebind<V>::other value_alloc_type; + + public: + typedef sparse_hashtable_iterator<V,K,HF,ExK,SetK,EqK,A> iterator; + typedef sparse_hashtable_const_iterator<V,K,HF,ExK,SetK,EqK,A> const_iterator; typedef typename sparsetable<V,DEFAULT_GROUP_SIZE,value_alloc_type>::const_nonempty_iterator - st_iterator; - - typedef std::forward_iterator_tag iterator_category; // very little defined! - typedef V value_type; - typedef typename value_alloc_type::difference_type difference_type; - typedef typename value_alloc_type::size_type size_type; - typedef typename value_alloc_type::const_reference reference; - typedef typename value_alloc_type::const_pointer pointer; - - // "Real" constructor and default constructor - sparse_hashtable_const_iterator(const sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *h, - st_iterator it, st_iterator it_end) - : ht(h), pos(it), end(it_end) { advance_past_deleted(); } - // This lets us convert regular iterators to const iterators - sparse_hashtable_const_iterator() { } // never used internally - sparse_hashtable_const_iterator(const iterator &it) - : ht(it.ht), pos(it.pos), end(it.end) { } - // The default destructor is fine; we don't define one - // The default operator= is fine; we don't define one - - // Happy dereferencer - reference operator*() const { return *pos; } - pointer operator->() const { return &(operator*()); } - - // Arithmetic. The only hard part is making sure that - // we're not on a marked-deleted array element - void advance_past_deleted() { - while ( pos != end && ht->test_deleted(*this) ) - ++pos; - } - const_iterator& operator++() { - assert(pos != end); ++pos; advance_past_deleted(); return *this; - } - const_iterator operator++(int) { const_iterator tmp(*this); ++*this; return tmp; } - - // Comparison. - bool operator==(const const_iterator& it) const { return pos == it.pos; } - bool operator!=(const const_iterator& it) const { return pos != it.pos; } - - - // The actual data - const sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *ht; - st_iterator pos, end; -}; - -// And once again, but this time freeing up memory as we iterate -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -struct sparse_hashtable_destructive_iterator { - private: - typedef typename A::template rebind<V>::other value_alloc_type; - - public: - typedef sparse_hashtable_destructive_iterator<V,K,HF,ExK,SetK,EqK,A> iterator; + st_iterator; + + typedef std::forward_iterator_tag iterator_category; // very little defined! + typedef V value_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::const_reference reference; + typedef typename value_alloc_type::const_pointer pointer; + + // "Real" constructor and default constructor + sparse_hashtable_const_iterator(const sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *h, + st_iterator it, st_iterator it_end) + : ht(h), pos(it), end(it_end) { advance_past_deleted(); } + // This lets us convert regular iterators to const iterators + sparse_hashtable_const_iterator() { } // never used internally + sparse_hashtable_const_iterator(const iterator &it) + : ht(it.ht), pos(it.pos), end(it.end) { } + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *pos; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic. The only hard part is making sure that + // we're not on a marked-deleted array element + void advance_past_deleted() { + while ( pos != end && ht->test_deleted(*this) ) + ++pos; + } + const_iterator& operator++() { + assert(pos != end); ++pos; advance_past_deleted(); return *this; + } + const_iterator operator++(int) { const_iterator tmp(*this); ++*this; return tmp; } + + // Comparison. + bool operator==(const const_iterator& it) const { return pos == it.pos; } + bool operator!=(const const_iterator& it) const { return pos != it.pos; } + + + // The actual data + const sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *ht; + st_iterator pos, end; +}; + +// And once again, but this time freeing up memory as we iterate +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +struct sparse_hashtable_destructive_iterator { + private: + typedef typename A::template rebind<V>::other value_alloc_type; + + public: + typedef sparse_hashtable_destructive_iterator<V,K,HF,ExK,SetK,EqK,A> iterator; typedef typename sparsetable<V,DEFAULT_GROUP_SIZE,value_alloc_type>::destructive_iterator - st_iterator; - - typedef std::forward_iterator_tag iterator_category; // very little defined! - typedef V value_type; - typedef typename value_alloc_type::difference_type difference_type; - typedef typename value_alloc_type::size_type size_type; - typedef typename value_alloc_type::reference reference; - typedef typename value_alloc_type::pointer pointer; - - // "Real" constructor and default constructor - sparse_hashtable_destructive_iterator(const - sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *h, - st_iterator it, st_iterator it_end) - : ht(h), pos(it), end(it_end) { advance_past_deleted(); } - sparse_hashtable_destructive_iterator() { } // never used internally - // The default destructor is fine; we don't define one - // The default operator= is fine; we don't define one - - // Happy dereferencer - reference operator*() const { return *pos; } - pointer operator->() const { return &(operator*()); } - - // Arithmetic. The only hard part is making sure that - // we're not on a marked-deleted array element - void advance_past_deleted() { - while ( pos != end && ht->test_deleted(*this) ) - ++pos; - } - iterator& operator++() { - assert(pos != end); ++pos; advance_past_deleted(); return *this; - } - iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } - - // Comparison. - bool operator==(const iterator& it) const { return pos == it.pos; } - bool operator!=(const iterator& it) const { return pos != it.pos; } - - - // The actual data - const sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *ht; - st_iterator pos, end; -}; - - -template <class Value, class Key, class HashFcn, - class ExtractKey, class SetKey, class EqualKey, class Alloc> -class sparse_hashtable { - private: - typedef typename Alloc::template rebind<Value>::other value_alloc_type; - - public: - typedef Key key_type; - typedef Value value_type; - typedef HashFcn hasher; - typedef EqualKey key_equal; - typedef Alloc allocator_type; - - typedef typename value_alloc_type::size_type size_type; - typedef typename value_alloc_type::difference_type difference_type; - typedef typename value_alloc_type::reference reference; - typedef typename value_alloc_type::const_reference const_reference; - typedef typename value_alloc_type::pointer pointer; - typedef typename value_alloc_type::const_pointer const_pointer; - typedef sparse_hashtable_iterator<Value, Key, HashFcn, ExtractKey, - SetKey, EqualKey, Alloc> - iterator; - - typedef sparse_hashtable_const_iterator<Value, Key, HashFcn, ExtractKey, - SetKey, EqualKey, Alloc> - const_iterator; - - typedef sparse_hashtable_destructive_iterator<Value, Key, HashFcn, ExtractKey, - SetKey, EqualKey, Alloc> - destructive_iterator; - - // These come from tr1. For us they're the same as regular iterators. - typedef iterator local_iterator; - typedef const_iterator const_local_iterator; - - // How full we let the table get before we resize, by default. - // Knuth says .8 is good -- higher causes us to probe too much, - // though it saves memory. - static const int HT_OCCUPANCY_PCT; // = 80 (out of 100); - - // How empty we let the table get before we resize lower, by default. - // (0.0 means never resize lower.) - // It should be less than OCCUPANCY_PCT / 2 or we thrash resizing - static const int HT_EMPTY_PCT; // = 0.4 * HT_OCCUPANCY_PCT; - - // Minimum size we're willing to let hashtables be. - // Must be a power of two, and at least 4. - // Note, however, that for a given hashtable, the initial size is a - // function of the first constructor arg, and may be >HT_MIN_BUCKETS. - static const size_type HT_MIN_BUCKETS = 4; - - // By default, if you don't specify a hashtable size at - // construction-time, we use this size. Must be a power of two, and - // at least HT_MIN_BUCKETS. - static const size_type HT_DEFAULT_STARTING_BUCKETS = 32; - - // ITERATOR FUNCTIONS - iterator begin() { return iterator(this, table.nonempty_begin(), - table.nonempty_end()); } - iterator end() { return iterator(this, table.nonempty_end(), - table.nonempty_end()); } - const_iterator begin() const { return const_iterator(this, - table.nonempty_begin(), - table.nonempty_end()); } - const_iterator end() const { return const_iterator(this, - table.nonempty_end(), - table.nonempty_end()); } - - // These come from tr1 unordered_map. They iterate over 'bucket' n. - // For sparsehashtable, we could consider each 'group' to be a bucket, - // I guess, but I don't really see the point. We'll just consider - // bucket n to be the n-th element of the sparsetable, if it's occupied, - // or some empty element, otherwise. - local_iterator begin(size_type i) { - if (table.test(i)) - return local_iterator(this, table.get_iter(i), table.nonempty_end()); - else - return local_iterator(this, table.nonempty_end(), table.nonempty_end()); - } - local_iterator end(size_type i) { - local_iterator it = begin(i); - if (table.test(i) && !test_deleted(i)) - ++it; - return it; - } - const_local_iterator begin(size_type i) const { - if (table.test(i)) - return const_local_iterator(this, table.get_iter(i), - table.nonempty_end()); - else - return const_local_iterator(this, table.nonempty_end(), - table.nonempty_end()); - } - const_local_iterator end(size_type i) const { - const_local_iterator it = begin(i); - if (table.test(i) && !test_deleted(i)) - ++it; - return it; - } - - // This is used when resizing - destructive_iterator destructive_begin() { - return destructive_iterator(this, table.destructive_begin(), - table.destructive_end()); - } - destructive_iterator destructive_end() { - return destructive_iterator(this, table.destructive_end(), - table.destructive_end()); - } - - - // ACCESSOR FUNCTIONS for the things we templatize on, basically - hasher hash_funct() const { return settings; } - key_equal key_eq() const { return key_info; } - allocator_type get_allocator() const { return table.get_allocator(); } - - // Accessor function for statistics gathering. - int num_table_copies() const { return settings.num_ht_copies(); } - - private: - // We need to copy values when we set the special marker for deleted - // elements, but, annoyingly, we can't just use the copy assignment - // operator because value_type might not be assignable (it's often - // pair<const X, Y>). We use explicit destructor invocation and - // placement new to get around this. Arg. - void set_value(pointer dst, const_reference src) { - dst->~value_type(); // delete the old value, if any - new(dst) value_type(src); - } - - // This is used as a tag for the copy constructor, saying to destroy its - // arg We have two ways of destructively copying: with potentially growing - // the hashtable as we copy, and without. To make sure the outside world - // can't do a destructive copy, we make the typename private. - enum MoveDontCopyT {MoveDontCopy, MoveDontGrow}; - - // DELETE HELPER FUNCTIONS - // This lets the user describe a key that will indicate deleted - // table entries. This key should be an "impossible" entry -- - // if you try to insert it for real, you won't be able to retrieve it! - // (NB: while you pass in an entire value, only the key part is looked - // at. This is just because I don't know how to assign just a key.) - private: - void squash_deleted() { // gets rid of any deleted entries we have - if ( num_deleted ) { // get rid of deleted before writing - sparse_hashtable tmp(MoveDontGrow, *this); - swap(tmp); // now we are tmp - } - assert(num_deleted == 0); - } - - // Test if the given key is the deleted indicator. Requires - // num_deleted > 0, for correctness of read(), and because that - // guarantees that key_info.delkey is valid. - bool test_deleted_key(const key_type& key) const { - assert(num_deleted > 0); - return equals(key_info.delkey, key); - } - - public: - void set_deleted_key(const key_type &key) { - // It's only safe to change what "deleted" means if we purge deleted guys - squash_deleted(); - settings.set_use_deleted(true); - key_info.delkey = key; - } - void clear_deleted_key() { - squash_deleted(); - settings.set_use_deleted(false); - } - key_type deleted_key() const { - assert(settings.use_deleted() - && "Must set deleted key before calling deleted_key"); - return key_info.delkey; - } - - // These are public so the iterators can use them - // True if the item at position bucknum is "deleted" marker - bool test_deleted(size_type bucknum) const { - // Invariant: !use_deleted() implies num_deleted is 0. - assert(settings.use_deleted() || num_deleted == 0); - return num_deleted > 0 && table.test(bucknum) && - test_deleted_key(get_key(table.unsafe_get(bucknum))); - } - bool test_deleted(const iterator &it) const { - // Invariant: !use_deleted() implies num_deleted is 0. - assert(settings.use_deleted() || num_deleted == 0); - return num_deleted > 0 && test_deleted_key(get_key(*it)); - } - bool test_deleted(const const_iterator &it) const { - // Invariant: !use_deleted() implies num_deleted is 0. - assert(settings.use_deleted() || num_deleted == 0); - return num_deleted > 0 && test_deleted_key(get_key(*it)); - } - bool test_deleted(const destructive_iterator &it) const { - // Invariant: !use_deleted() implies num_deleted is 0. - assert(settings.use_deleted() || num_deleted == 0); - return num_deleted > 0 && test_deleted_key(get_key(*it)); - } - - private: - void check_use_deleted(const char* caller) { - (void)caller; // could log it if the assert failed - assert(settings.use_deleted()); - } - - // Set it so test_deleted is true. true if object didn't used to be deleted. - // TODO(csilvers): make these private (also in densehashtable.h) - bool set_deleted(iterator &it) { - check_use_deleted("set_deleted()"); - bool retval = !test_deleted(it); - // &* converts from iterator to value-type. - set_key(&(*it), key_info.delkey); - return retval; - } - // Set it so test_deleted is false. true if object used to be deleted. - bool clear_deleted(iterator &it) { - check_use_deleted("clear_deleted()"); - // Happens automatically when we assign something else in its place. - return test_deleted(it); - } - - // We also allow to set/clear the deleted bit on a const iterator. - // We allow a const_iterator for the same reason you can delete a - // const pointer: it's convenient, and semantically you can't use - // 'it' after it's been deleted anyway, so its const-ness doesn't - // really matter. - bool set_deleted(const_iterator &it) { - check_use_deleted("set_deleted()"); - bool retval = !test_deleted(it); - set_key(const_cast<pointer>(&(*it)), key_info.delkey); - return retval; - } - // Set it so test_deleted is false. true if object used to be deleted. - bool clear_deleted(const_iterator &it) { - check_use_deleted("clear_deleted()"); - return test_deleted(it); - } - - // FUNCTIONS CONCERNING SIZE - public: - size_type size() const { return table.num_nonempty() - num_deleted; } - size_type max_size() const { return table.max_size(); } - bool empty() const { return size() == 0; } - size_type bucket_count() const { return table.size(); } - size_type max_bucket_count() const { return max_size(); } - // These are tr1 methods. Their idea of 'bucket' doesn't map well to - // what we do. We just say every bucket has 0 or 1 items in it. - size_type bucket_size(size_type i) const { - return begin(i) == end(i) ? 0 : 1; - } - - private: - // Because of the above, size_type(-1) is never legal; use it for errors - static const size_type ILLEGAL_BUCKET = size_type(-1); - - // Used after a string of deletes. Returns true if we actually shrunk. - // TODO(csilvers): take a delta so we can take into account inserts - // done after shrinking. Maybe make part of the Settings class? - bool maybe_shrink() { - assert(table.num_nonempty() >= num_deleted); - assert((bucket_count() & (bucket_count()-1)) == 0); // is a power of two - assert(bucket_count() >= HT_MIN_BUCKETS); - bool retval = false; - - // If you construct a hashtable with < HT_DEFAULT_STARTING_BUCKETS, - // we'll never shrink until you get relatively big, and we'll never - // shrink below HT_DEFAULT_STARTING_BUCKETS. Otherwise, something - // like "dense_hash_set<int> x; x.insert(4); x.erase(4);" will - // shrink us down to HT_MIN_BUCKETS buckets, which is too small. - const size_type num_remain = table.num_nonempty() - num_deleted; - const size_type shrink_threshold = settings.shrink_threshold(); - if (shrink_threshold > 0 && num_remain < shrink_threshold && - bucket_count() > HT_DEFAULT_STARTING_BUCKETS) { - const float shrink_factor = settings.shrink_factor(); - size_type sz = bucket_count() / 2; // find how much we should shrink - while (sz > HT_DEFAULT_STARTING_BUCKETS && - num_remain < static_cast<size_type>(sz * shrink_factor)) { - sz /= 2; // stay a power of 2 - } - sparse_hashtable tmp(MoveDontCopy, *this, sz); - swap(tmp); // now we are tmp - retval = true; - } - settings.set_consider_shrink(false); // because we just considered it - return retval; - } - - // We'll let you resize a hashtable -- though this makes us copy all! - // When you resize, you say, "make it big enough for this many more elements" - // Returns true if we actually resized, false if size was already ok. - bool resize_delta(size_type delta) { - bool did_resize = false; - if ( settings.consider_shrink() ) { // see if lots of deletes happened - if ( maybe_shrink() ) - did_resize = true; - } - if (table.num_nonempty() >= - (std::numeric_limits<size_type>::max)() - delta) { - throw std::length_error("resize overflow"); - } - if ( bucket_count() >= HT_MIN_BUCKETS && - (table.num_nonempty() + delta) <= settings.enlarge_threshold() ) - return did_resize; // we're ok as we are - - // Sometimes, we need to resize just to get rid of all the - // "deleted" buckets that are clogging up the hashtable. So when - // deciding whether to resize, count the deleted buckets (which - // are currently taking up room). But later, when we decide what - // size to resize to, *don't* count deleted buckets, since they - // get discarded during the resize. - const size_type needed_size = - settings.min_buckets(table.num_nonempty() + delta, 0); - if ( needed_size <= bucket_count() ) // we have enough buckets - return did_resize; - - size_type resize_to = - settings.min_buckets(table.num_nonempty() - num_deleted + delta, - bucket_count()); - if (resize_to < needed_size && // may double resize_to - resize_to < (std::numeric_limits<size_type>::max)() / 2) { - // This situation means that we have enough deleted elements, - // that once we purge them, we won't actually have needed to - // grow. But we may want to grow anyway: if we just purge one - // element, say, we'll have to grow anyway next time we - // insert. Might as well grow now, since we're already going - // through the trouble of copying (in order to purge the - // deleted elements). - const size_type target = - static_cast<size_type>(settings.shrink_size(resize_to*2)); - if (table.num_nonempty() - num_deleted + delta >= target) { - // Good, we won't be below the shrink threshhold even if we double. - resize_to *= 2; - } - } - - sparse_hashtable tmp(MoveDontCopy, *this, resize_to); - swap(tmp); // now we are tmp - return true; - } - - // Used to actually do the rehashing when we grow/shrink a hashtable - void copy_from(const sparse_hashtable &ht, size_type min_buckets_wanted) { - clear(); // clear table, set num_deleted to 0 - - // If we need to change the size of our table, do it now - const size_type resize_to = - settings.min_buckets(ht.size(), min_buckets_wanted); - if ( resize_to > bucket_count() ) { // we don't have enough buckets - table.resize(resize_to); // sets the number of buckets - settings.reset_thresholds(bucket_count()); - } - - // We use a normal iterator to get non-deleted bcks from ht - // We could use insert() here, but since we know there are - // no duplicates and no deleted items, we can be more efficient - assert((bucket_count() & (bucket_count()-1)) == 0); // a power of two - for ( const_iterator it = ht.begin(); it != ht.end(); ++it ) { - size_type num_probes = 0; // how many times we've probed - size_type bucknum; - const size_type bucket_count_minus_one = bucket_count() - 1; - for (bucknum = hash(get_key(*it)) & bucket_count_minus_one; - table.test(bucknum); // not empty - bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one) { - ++num_probes; - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - table.set(bucknum, *it); // copies the value to here - } - settings.inc_num_ht_copies(); - } - - // Implementation is like copy_from, but it destroys the table of the - // "from" guy by freeing sparsetable memory as we iterate. This is - // useful in resizing, since we're throwing away the "from" guy anyway. - void move_from(MoveDontCopyT mover, sparse_hashtable &ht, - size_type min_buckets_wanted) { - clear(); // clear table, set num_deleted to 0 - - // If we need to change the size of our table, do it now - size_type resize_to; - if ( mover == MoveDontGrow ) - resize_to = ht.bucket_count(); // keep same size as old ht - else // MoveDontCopy - resize_to = settings.min_buckets(ht.size(), min_buckets_wanted); - if ( resize_to > bucket_count() ) { // we don't have enough buckets - table.resize(resize_to); // sets the number of buckets - settings.reset_thresholds(bucket_count()); - } - - // We use a normal iterator to get non-deleted bcks from ht - // We could use insert() here, but since we know there are - // no duplicates and no deleted items, we can be more efficient - assert( (bucket_count() & (bucket_count()-1)) == 0); // a power of two - // THIS IS THE MAJOR LINE THAT DIFFERS FROM COPY_FROM(): - for ( destructive_iterator it = ht.destructive_begin(); - it != ht.destructive_end(); ++it ) { - size_type num_probes = 0; // how many times we've probed - size_type bucknum; - for ( bucknum = hash(get_key(*it)) & (bucket_count()-1); // h % buck_cnt - table.test(bucknum); // not empty - bucknum = (bucknum + JUMP_(key, num_probes)) & (bucket_count()-1) ) { - ++num_probes; - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - table.set(bucknum, *it); // copies the value to here - } - settings.inc_num_ht_copies(); - } - - - // Required by the spec for hashed associative container - public: - // Though the docs say this should be num_buckets, I think it's much - // more useful as num_elements. As a special feature, calling with - // req_elements==0 will cause us to shrink if we can, saving space. - void resize(size_type req_elements) { // resize to this or larger - if ( settings.consider_shrink() || req_elements == 0 ) - maybe_shrink(); - if ( req_elements > table.num_nonempty() ) // we only grow - resize_delta(req_elements - table.num_nonempty()); - } - - // Get and change the value of shrink_factor and enlarge_factor. The - // description at the beginning of this file explains how to choose - // the values. Setting the shrink parameter to 0.0 ensures that the - // table never shrinks. - void get_resizing_parameters(float* shrink, float* grow) const { - *shrink = settings.shrink_factor(); - *grow = settings.enlarge_factor(); - } - void set_resizing_parameters(float shrink, float grow) { - settings.set_resizing_parameters(shrink, grow); - settings.reset_thresholds(bucket_count()); - } - - // CONSTRUCTORS -- as required by the specs, we take a size, - // but also let you specify a hashfunction, key comparator, - // and key extractor. We also define a copy constructor and =. - // DESTRUCTOR -- the default is fine, surprisingly. - explicit sparse_hashtable(size_type expected_max_items_in_table = 0, - const HashFcn& hf = HashFcn(), - const EqualKey& eql = EqualKey(), - const ExtractKey& ext = ExtractKey(), - const SetKey& set = SetKey(), - const Alloc& alloc = Alloc()) - : settings(hf), - key_info(ext, set, eql), - num_deleted(0), - table((expected_max_items_in_table == 0 - ? HT_DEFAULT_STARTING_BUCKETS - : settings.min_buckets(expected_max_items_in_table, 0)), - alloc) { - settings.reset_thresholds(bucket_count()); - } - - // As a convenience for resize(), we allow an optional second argument - // which lets you make this new hashtable a different size than ht. - // We also provide a mechanism of saying you want to "move" the ht argument - // into us instead of copying. - sparse_hashtable(const sparse_hashtable& ht, - size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS) - : settings(ht.settings), - key_info(ht.key_info), - num_deleted(0), - table(0, ht.get_allocator()) { - settings.reset_thresholds(bucket_count()); - copy_from(ht, min_buckets_wanted); // copy_from() ignores deleted entries - } - sparse_hashtable(MoveDontCopyT mover, sparse_hashtable& ht, - size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS) - : settings(ht.settings), - key_info(ht.key_info), - num_deleted(0), - table(0, ht.get_allocator()) { - settings.reset_thresholds(bucket_count()); - move_from(mover, ht, min_buckets_wanted); // ignores deleted entries - } - - sparse_hashtable& operator= (const sparse_hashtable& ht) { - if (&ht == this) return *this; // don't copy onto ourselves - settings = ht.settings; - key_info = ht.key_info; - num_deleted = ht.num_deleted; - // copy_from() calls clear and sets num_deleted to 0 too - copy_from(ht, HT_MIN_BUCKETS); - // we purposefully don't copy the allocator, which may not be copyable - return *this; - } - - // Many STL algorithms use swap instead of copy constructors - void swap(sparse_hashtable& ht) { - std::swap(settings, ht.settings); - std::swap(key_info, ht.key_info); - std::swap(num_deleted, ht.num_deleted); - table.swap(ht.table); - settings.reset_thresholds(bucket_count()); // also resets consider_shrink - ht.settings.reset_thresholds(ht.bucket_count()); - // we purposefully don't swap the allocator, which may not be swap-able - } - - // It's always nice to be able to clear a table without deallocating it - void clear() { - if (!empty() || (num_deleted != 0)) { - table.clear(); - } - settings.reset_thresholds(bucket_count()); - num_deleted = 0; - } - - // LOOKUP ROUTINES - private: - // Returns a pair of positions: 1st where the object is, 2nd where - // it would go if you wanted to insert it. 1st is ILLEGAL_BUCKET - // if object is not found; 2nd is ILLEGAL_BUCKET if it is. - // Note: because of deletions where-to-insert is not trivial: it's the - // first deleted bucket we see, as long as we don't find the key later - std::pair<size_type, size_type> find_position(const key_type &key) const { - size_type num_probes = 0; // how many times we've probed - const size_type bucket_count_minus_one = bucket_count() - 1; - size_type bucknum = hash(key) & bucket_count_minus_one; - size_type insert_pos = ILLEGAL_BUCKET; // where we would insert - SPARSEHASH_STAT_UPDATE(total_lookups += 1); - while ( 1 ) { // probe until something happens - if ( !table.test(bucknum) ) { // bucket is empty - SPARSEHASH_STAT_UPDATE(total_probes += num_probes); - if ( insert_pos == ILLEGAL_BUCKET ) // found no prior place to insert - return std::pair<size_type,size_type>(ILLEGAL_BUCKET, bucknum); - else - return std::pair<size_type,size_type>(ILLEGAL_BUCKET, insert_pos); - - } else if ( test_deleted(bucknum) ) {// keep searching, but mark to insert - if ( insert_pos == ILLEGAL_BUCKET ) - insert_pos = bucknum; - - } else if ( equals(key, get_key(table.unsafe_get(bucknum))) ) { - SPARSEHASH_STAT_UPDATE(total_probes += num_probes); - return std::pair<size_type,size_type>(bucknum, ILLEGAL_BUCKET); - } - ++num_probes; // we're doing another probe - bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one; - assert(num_probes < bucket_count() - && "Hashtable is full: an error in key_equal<> or hash<>"); - } - } - - public: - - iterator find(const key_type& key) { - if ( size() == 0 ) return end(); - std::pair<size_type, size_type> pos = find_position(key); - if ( pos.first == ILLEGAL_BUCKET ) // alas, not there - return end(); - else - return iterator(this, table.get_iter(pos.first), table.nonempty_end()); - } - - const_iterator find(const key_type& key) const { - if ( size() == 0 ) return end(); - std::pair<size_type, size_type> pos = find_position(key); - if ( pos.first == ILLEGAL_BUCKET ) // alas, not there - return end(); - else - return const_iterator(this, - table.get_iter(pos.first), table.nonempty_end()); - } - - // This is a tr1 method: the bucket a given key is in, or what bucket - // it would be put in, if it were to be inserted. Shrug. - size_type bucket(const key_type& key) const { - std::pair<size_type, size_type> pos = find_position(key); - return pos.first == ILLEGAL_BUCKET ? pos.second : pos.first; - } - - // Counts how many elements have key key. For maps, it's either 0 or 1. - size_type count(const key_type &key) const { - std::pair<size_type, size_type> pos = find_position(key); - return pos.first == ILLEGAL_BUCKET ? 0 : 1; - } - - // Likewise, equal_range doesn't really make sense for us. Oh well. - std::pair<iterator,iterator> equal_range(const key_type& key) { - iterator pos = find(key); // either an iterator or end - if (pos == end()) { - return std::pair<iterator,iterator>(pos, pos); - } else { - const iterator startpos = pos++; - return std::pair<iterator,iterator>(startpos, pos); - } - } - std::pair<const_iterator,const_iterator> equal_range(const key_type& key) - const { - const_iterator pos = find(key); // either an iterator or end - if (pos == end()) { - return std::pair<const_iterator,const_iterator>(pos, pos); - } else { - const const_iterator startpos = pos++; - return std::pair<const_iterator,const_iterator>(startpos, pos); - } - } - - - // INSERTION ROUTINES - private: - // Private method used by insert_noresize and find_or_insert. - iterator insert_at(const_reference obj, size_type pos) { - if (size() >= max_size()) { - throw std::length_error("insert overflow"); - } - if ( test_deleted(pos) ) { // just replace if it's been deleted - // The set() below will undelete this object. We just worry about stats - assert(num_deleted > 0); - --num_deleted; // used to be, now it isn't - } - table.set(pos, obj); - return iterator(this, table.get_iter(pos), table.nonempty_end()); - } - - // If you know *this is big enough to hold obj, use this routine - std::pair<iterator, bool> insert_noresize(const_reference obj) { - // First, double-check we're not inserting delkey - assert((!settings.use_deleted() || !equals(get_key(obj), key_info.delkey)) - && "Inserting the deleted key"); - const std::pair<size_type,size_type> pos = find_position(get_key(obj)); - if ( pos.first != ILLEGAL_BUCKET) { // object was already there - return std::pair<iterator,bool>(iterator(this, table.get_iter(pos.first), - table.nonempty_end()), - false); // false: we didn't insert - } else { // pos.second says where to put it - return std::pair<iterator,bool>(insert_at(obj, pos.second), true); - } - } - - // Specializations of insert(it, it) depending on the power of the iterator: - // (1) Iterator supports operator-, resize before inserting - template <class ForwardIterator> - void insert(ForwardIterator f, ForwardIterator l, std::forward_iterator_tag) { - size_t dist = std::distance(f, l); - if (dist >= (std::numeric_limits<size_type>::max)()) { - throw std::length_error("insert-range overflow"); - } - resize_delta(static_cast<size_type>(dist)); - for ( ; dist > 0; --dist, ++f) { - insert_noresize(*f); - } - } - - // (2) Arbitrary iterator, can't tell how much to resize - template <class InputIterator> - void insert(InputIterator f, InputIterator l, std::input_iterator_tag) { - for ( ; f != l; ++f) - insert(*f); - } - - public: - // This is the normal insert routine, used by the outside world - std::pair<iterator, bool> insert(const_reference obj) { - resize_delta(1); // adding an object, grow if need be - return insert_noresize(obj); - } - - // When inserting a lot at a time, we specialize on the type of iterator - template <class InputIterator> - void insert(InputIterator f, InputIterator l) { - // specializes on iterator type - insert(f, l, - typename std::iterator_traits<InputIterator>::iterator_category()); - } - - // DefaultValue is a functor that takes a key and returns a value_type - // representing the default value to be inserted if none is found. - template <class DefaultValue> - value_type& find_or_insert(const key_type& key) { - // First, double-check we're not inserting delkey - assert((!settings.use_deleted() || !equals(key, key_info.delkey)) - && "Inserting the deleted key"); - const std::pair<size_type,size_type> pos = find_position(key); - DefaultValue default_value; - if ( pos.first != ILLEGAL_BUCKET) { // object was already there - return *table.get_iter(pos.first); - } else if (resize_delta(1)) { // needed to rehash to make room - // Since we resized, we can't use pos, so recalculate where to insert. - return *insert_noresize(default_value(key)).first; - } else { // no need to rehash, insert right here - return *insert_at(default_value(key), pos.second); - } - } - - // DELETION ROUTINES - size_type erase(const key_type& key) { - // First, double-check we're not erasing delkey. - assert((!settings.use_deleted() || !equals(key, key_info.delkey)) - && "Erasing the deleted key"); - assert(!settings.use_deleted() || !equals(key, key_info.delkey)); - const_iterator pos = find(key); // shrug: shouldn't need to be const - if ( pos != end() ) { - assert(!test_deleted(pos)); // or find() shouldn't have returned it - set_deleted(pos); - ++num_deleted; - // will think about shrink after next insert - settings.set_consider_shrink(true); - return 1; // because we deleted one thing - } else { - return 0; // because we deleted nothing - } - } - - // We return the iterator past the deleted item. - void erase(iterator pos) { - if ( pos == end() ) return; // sanity check - if ( set_deleted(pos) ) { // true if object has been newly deleted - ++num_deleted; - // will think about shrink after next insert - settings.set_consider_shrink(true); - } - } - - void erase(iterator f, iterator l) { - for ( ; f != l; ++f) { - if ( set_deleted(f) ) // should always be true - ++num_deleted; - } - // will think about shrink after next insert - settings.set_consider_shrink(true); - } - - // We allow you to erase a const_iterator just like we allow you to - // erase an iterator. This is in parallel to 'delete': you can delete - // a const pointer just like a non-const pointer. The logic is that - // you can't use the object after it's erased anyway, so it doesn't matter - // if it's const or not. - void erase(const_iterator pos) { - if ( pos == end() ) return; // sanity check - if ( set_deleted(pos) ) { // true if object has been newly deleted - ++num_deleted; - // will think about shrink after next insert - settings.set_consider_shrink(true); - } - } - void erase(const_iterator f, const_iterator l) { - for ( ; f != l; ++f) { - if ( set_deleted(f) ) // should always be true - ++num_deleted; - } - // will think about shrink after next insert - settings.set_consider_shrink(true); - } - - - // COMPARISON - bool operator==(const sparse_hashtable& ht) const { - if (size() != ht.size()) { - return false; - } else if (this == &ht) { - return true; - } else { - // Iterate through the elements in "this" and see if the - // corresponding element is in ht - for ( const_iterator it = begin(); it != end(); ++it ) { - const_iterator it2 = ht.find(get_key(*it)); - if ((it2 == ht.end()) || (*it != *it2)) { - return false; - } - } - return true; - } - } - bool operator!=(const sparse_hashtable& ht) const { - return !(*this == ht); - } - - - // I/O - // We support reading and writing hashtables to disk. NOTE that - // this only stores the hashtable metadata, not the stuff you've - // actually put in the hashtable! Alas, since I don't know how to - // write a hasher or key_equal, you have to make sure everything - // but the table is the same. We compact before writing. - // - // The OUTPUT type needs to support a Write() operation. File and - // OutputBuffer are appropriate types to pass in. - // - // The INPUT type needs to support a Read() operation. File and - // InputBuffer are appropriate types to pass in. - template <typename OUTPUT> - bool write_metadata(OUTPUT *fp) { - squash_deleted(); // so we don't have to worry about delkey - return table.write_metadata(fp); - } - - template <typename INPUT> - bool read_metadata(INPUT *fp) { - num_deleted = 0; // since we got rid before writing - const bool result = table.read_metadata(fp); - settings.reset_thresholds(bucket_count()); - return result; - } - - // Only meaningful if value_type is a POD. - template <typename OUTPUT> - bool write_nopointer_data(OUTPUT *fp) { - return table.write_nopointer_data(fp); - } - - // Only meaningful if value_type is a POD. - template <typename INPUT> - bool read_nopointer_data(INPUT *fp) { - return table.read_nopointer_data(fp); - } - - // INPUT and OUTPUT must be either a FILE, *or* a C++ stream - // (istream, ostream, etc) *or* a class providing - // Read(void*, size_t) and Write(const void*, size_t) - // (respectively), which writes a buffer into a stream - // (which the INPUT/OUTPUT instance presumably owns). - - typedef sparsehash_internal::pod_serializer<value_type> NopointerSerializer; - - // ValueSerializer: a functor. operator()(OUTPUT*, const value_type&) - template <typename ValueSerializer, typename OUTPUT> - bool serialize(ValueSerializer serializer, OUTPUT *fp) { - squash_deleted(); // so we don't have to worry about delkey - return table.serialize(serializer, fp); - } - - // ValueSerializer: a functor. operator()(INPUT*, value_type*) - template <typename ValueSerializer, typename INPUT> - bool unserialize(ValueSerializer serializer, INPUT *fp) { - num_deleted = 0; // since we got rid before writing - const bool result = table.unserialize(serializer, fp); - settings.reset_thresholds(bucket_count()); - return result; - } - - private: - // Table is the main storage class. - typedef sparsetable<value_type, DEFAULT_GROUP_SIZE, value_alloc_type> Table; - - // Package templated functors with the other types to eliminate memory - // needed for storing these zero-size operators. Since ExtractKey and - // hasher's operator() might have the same function signature, they - // must be packaged in different classes. - struct Settings : - sparsehash_internal::sh_hashtable_settings<key_type, hasher, - size_type, HT_MIN_BUCKETS> { - explicit Settings(const hasher& hf) - : sparsehash_internal::sh_hashtable_settings<key_type, hasher, - size_type, HT_MIN_BUCKETS>( - hf, HT_OCCUPANCY_PCT / 100.0f, HT_EMPTY_PCT / 100.0f) {} - }; - - // KeyInfo stores delete key and packages zero-size functors: - // ExtractKey and SetKey. - class KeyInfo : public ExtractKey, public SetKey, public EqualKey { - public: - KeyInfo(const ExtractKey& ek, const SetKey& sk, const EqualKey& eq) - : ExtractKey(ek), - SetKey(sk), - EqualKey(eq) { - } - // We want to return the exact same type as ExtractKey: Key or const Key& - typename ExtractKey::result_type get_key(const_reference v) const { - return ExtractKey::operator()(v); - } - void set_key(pointer v, const key_type& k) const { - SetKey::operator()(v, k); - } - bool equals(const key_type& a, const key_type& b) const { - return EqualKey::operator()(a, b); - } - - // Which key marks deleted entries. - // TODO(csilvers): make a pointer, and get rid of use_deleted (benchmark!) - typename base::remove_const<key_type>::type delkey; - }; - - // Utility functions to access the templated operators - size_type hash(const key_type& v) const { - return settings.hash(v); - } - bool equals(const key_type& a, const key_type& b) const { - return key_info.equals(a, b); - } - typename ExtractKey::result_type get_key(const_reference v) const { - return key_info.get_key(v); - } - void set_key(pointer v, const key_type& k) const { - key_info.set_key(v, k); - } - - private: - // Actual data - Settings settings; - KeyInfo key_info; - size_type num_deleted; // how many occupied buckets are marked deleted - Table table; // holds num_buckets and num_elements too -}; - - -// We need a global swap as well -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -inline void swap(sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> &x, - sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> &y) { - x.swap(y); -} - -#undef JUMP_ - -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -const typename sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::size_type - sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::ILLEGAL_BUCKET; - -// How full we let the table get before we resize. Knuth says .8 is -// good -- higher causes us to probe too much, though saves memory -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -const int sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT = 80; - -// How empty we let the table get before we resize lower. -// It should be less than OCCUPANCY_PCT / 2 or we thrash resizing -template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> -const int sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_EMPTY_PCT - = static_cast<int>(0.4 * - sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT); - -_END_GOOGLE_NAMESPACE_ - -#endif /* _SPARSEHASHTABLE_H_ */ + st_iterator; + + typedef std::forward_iterator_tag iterator_category; // very little defined! + typedef V value_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::pointer pointer; + + // "Real" constructor and default constructor + sparse_hashtable_destructive_iterator(const + sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *h, + st_iterator it, st_iterator it_end) + : ht(h), pos(it), end(it_end) { advance_past_deleted(); } + sparse_hashtable_destructive_iterator() { } // never used internally + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *pos; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic. The only hard part is making sure that + // we're not on a marked-deleted array element + void advance_past_deleted() { + while ( pos != end && ht->test_deleted(*this) ) + ++pos; + } + iterator& operator++() { + assert(pos != end); ++pos; advance_past_deleted(); return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + + // Comparison. + bool operator==(const iterator& it) const { return pos == it.pos; } + bool operator!=(const iterator& it) const { return pos != it.pos; } + + + // The actual data + const sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> *ht; + st_iterator pos, end; +}; + + +template <class Value, class Key, class HashFcn, + class ExtractKey, class SetKey, class EqualKey, class Alloc> +class sparse_hashtable { + private: + typedef typename Alloc::template rebind<Value>::other value_alloc_type; + + public: + typedef Key key_type; + typedef Value value_type; + typedef HashFcn hasher; + typedef EqualKey key_equal; + typedef Alloc allocator_type; + + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::const_reference const_reference; + typedef typename value_alloc_type::pointer pointer; + typedef typename value_alloc_type::const_pointer const_pointer; + typedef sparse_hashtable_iterator<Value, Key, HashFcn, ExtractKey, + SetKey, EqualKey, Alloc> + iterator; + + typedef sparse_hashtable_const_iterator<Value, Key, HashFcn, ExtractKey, + SetKey, EqualKey, Alloc> + const_iterator; + + typedef sparse_hashtable_destructive_iterator<Value, Key, HashFcn, ExtractKey, + SetKey, EqualKey, Alloc> + destructive_iterator; + + // These come from tr1. For us they're the same as regular iterators. + typedef iterator local_iterator; + typedef const_iterator const_local_iterator; + + // How full we let the table get before we resize, by default. + // Knuth says .8 is good -- higher causes us to probe too much, + // though it saves memory. + static const int HT_OCCUPANCY_PCT; // = 80 (out of 100); + + // How empty we let the table get before we resize lower, by default. + // (0.0 means never resize lower.) + // It should be less than OCCUPANCY_PCT / 2 or we thrash resizing + static const int HT_EMPTY_PCT; // = 0.4 * HT_OCCUPANCY_PCT; + + // Minimum size we're willing to let hashtables be. + // Must be a power of two, and at least 4. + // Note, however, that for a given hashtable, the initial size is a + // function of the first constructor arg, and may be >HT_MIN_BUCKETS. + static const size_type HT_MIN_BUCKETS = 4; + + // By default, if you don't specify a hashtable size at + // construction-time, we use this size. Must be a power of two, and + // at least HT_MIN_BUCKETS. + static const size_type HT_DEFAULT_STARTING_BUCKETS = 32; + + // ITERATOR FUNCTIONS + iterator begin() { return iterator(this, table.nonempty_begin(), + table.nonempty_end()); } + iterator end() { return iterator(this, table.nonempty_end(), + table.nonempty_end()); } + const_iterator begin() const { return const_iterator(this, + table.nonempty_begin(), + table.nonempty_end()); } + const_iterator end() const { return const_iterator(this, + table.nonempty_end(), + table.nonempty_end()); } + + // These come from tr1 unordered_map. They iterate over 'bucket' n. + // For sparsehashtable, we could consider each 'group' to be a bucket, + // I guess, but I don't really see the point. We'll just consider + // bucket n to be the n-th element of the sparsetable, if it's occupied, + // or some empty element, otherwise. + local_iterator begin(size_type i) { + if (table.test(i)) + return local_iterator(this, table.get_iter(i), table.nonempty_end()); + else + return local_iterator(this, table.nonempty_end(), table.nonempty_end()); + } + local_iterator end(size_type i) { + local_iterator it = begin(i); + if (table.test(i) && !test_deleted(i)) + ++it; + return it; + } + const_local_iterator begin(size_type i) const { + if (table.test(i)) + return const_local_iterator(this, table.get_iter(i), + table.nonempty_end()); + else + return const_local_iterator(this, table.nonempty_end(), + table.nonempty_end()); + } + const_local_iterator end(size_type i) const { + const_local_iterator it = begin(i); + if (table.test(i) && !test_deleted(i)) + ++it; + return it; + } + + // This is used when resizing + destructive_iterator destructive_begin() { + return destructive_iterator(this, table.destructive_begin(), + table.destructive_end()); + } + destructive_iterator destructive_end() { + return destructive_iterator(this, table.destructive_end(), + table.destructive_end()); + } + + + // ACCESSOR FUNCTIONS for the things we templatize on, basically + hasher hash_funct() const { return settings; } + key_equal key_eq() const { return key_info; } + allocator_type get_allocator() const { return table.get_allocator(); } + + // Accessor function for statistics gathering. + int num_table_copies() const { return settings.num_ht_copies(); } + + private: + // We need to copy values when we set the special marker for deleted + // elements, but, annoyingly, we can't just use the copy assignment + // operator because value_type might not be assignable (it's often + // pair<const X, Y>). We use explicit destructor invocation and + // placement new to get around this. Arg. + void set_value(pointer dst, const_reference src) { + dst->~value_type(); // delete the old value, if any + new(dst) value_type(src); + } + + // This is used as a tag for the copy constructor, saying to destroy its + // arg We have two ways of destructively copying: with potentially growing + // the hashtable as we copy, and without. To make sure the outside world + // can't do a destructive copy, we make the typename private. + enum MoveDontCopyT {MoveDontCopy, MoveDontGrow}; + + // DELETE HELPER FUNCTIONS + // This lets the user describe a key that will indicate deleted + // table entries. This key should be an "impossible" entry -- + // if you try to insert it for real, you won't be able to retrieve it! + // (NB: while you pass in an entire value, only the key part is looked + // at. This is just because I don't know how to assign just a key.) + private: + void squash_deleted() { // gets rid of any deleted entries we have + if ( num_deleted ) { // get rid of deleted before writing + sparse_hashtable tmp(MoveDontGrow, *this); + swap(tmp); // now we are tmp + } + assert(num_deleted == 0); + } + + // Test if the given key is the deleted indicator. Requires + // num_deleted > 0, for correctness of read(), and because that + // guarantees that key_info.delkey is valid. + bool test_deleted_key(const key_type& key) const { + assert(num_deleted > 0); + return equals(key_info.delkey, key); + } + + public: + void set_deleted_key(const key_type &key) { + // It's only safe to change what "deleted" means if we purge deleted guys + squash_deleted(); + settings.set_use_deleted(true); + key_info.delkey = key; + } + void clear_deleted_key() { + squash_deleted(); + settings.set_use_deleted(false); + } + key_type deleted_key() const { + assert(settings.use_deleted() + && "Must set deleted key before calling deleted_key"); + return key_info.delkey; + } + + // These are public so the iterators can use them + // True if the item at position bucknum is "deleted" marker + bool test_deleted(size_type bucknum) const { + // Invariant: !use_deleted() implies num_deleted is 0. + assert(settings.use_deleted() || num_deleted == 0); + return num_deleted > 0 && table.test(bucknum) && + test_deleted_key(get_key(table.unsafe_get(bucknum))); + } + bool test_deleted(const iterator &it) const { + // Invariant: !use_deleted() implies num_deleted is 0. + assert(settings.use_deleted() || num_deleted == 0); + return num_deleted > 0 && test_deleted_key(get_key(*it)); + } + bool test_deleted(const const_iterator &it) const { + // Invariant: !use_deleted() implies num_deleted is 0. + assert(settings.use_deleted() || num_deleted == 0); + return num_deleted > 0 && test_deleted_key(get_key(*it)); + } + bool test_deleted(const destructive_iterator &it) const { + // Invariant: !use_deleted() implies num_deleted is 0. + assert(settings.use_deleted() || num_deleted == 0); + return num_deleted > 0 && test_deleted_key(get_key(*it)); + } + + private: + void check_use_deleted(const char* caller) { + (void)caller; // could log it if the assert failed + assert(settings.use_deleted()); + } + + // Set it so test_deleted is true. true if object didn't used to be deleted. + // TODO(csilvers): make these private (also in densehashtable.h) + bool set_deleted(iterator &it) { + check_use_deleted("set_deleted()"); + bool retval = !test_deleted(it); + // &* converts from iterator to value-type. + set_key(&(*it), key_info.delkey); + return retval; + } + // Set it so test_deleted is false. true if object used to be deleted. + bool clear_deleted(iterator &it) { + check_use_deleted("clear_deleted()"); + // Happens automatically when we assign something else in its place. + return test_deleted(it); + } + + // We also allow to set/clear the deleted bit on a const iterator. + // We allow a const_iterator for the same reason you can delete a + // const pointer: it's convenient, and semantically you can't use + // 'it' after it's been deleted anyway, so its const-ness doesn't + // really matter. + bool set_deleted(const_iterator &it) { + check_use_deleted("set_deleted()"); + bool retval = !test_deleted(it); + set_key(const_cast<pointer>(&(*it)), key_info.delkey); + return retval; + } + // Set it so test_deleted is false. true if object used to be deleted. + bool clear_deleted(const_iterator &it) { + check_use_deleted("clear_deleted()"); + return test_deleted(it); + } + + // FUNCTIONS CONCERNING SIZE + public: + size_type size() const { return table.num_nonempty() - num_deleted; } + size_type max_size() const { return table.max_size(); } + bool empty() const { return size() == 0; } + size_type bucket_count() const { return table.size(); } + size_type max_bucket_count() const { return max_size(); } + // These are tr1 methods. Their idea of 'bucket' doesn't map well to + // what we do. We just say every bucket has 0 or 1 items in it. + size_type bucket_size(size_type i) const { + return begin(i) == end(i) ? 0 : 1; + } + + private: + // Because of the above, size_type(-1) is never legal; use it for errors + static const size_type ILLEGAL_BUCKET = size_type(-1); + + // Used after a string of deletes. Returns true if we actually shrunk. + // TODO(csilvers): take a delta so we can take into account inserts + // done after shrinking. Maybe make part of the Settings class? + bool maybe_shrink() { + assert(table.num_nonempty() >= num_deleted); + assert((bucket_count() & (bucket_count()-1)) == 0); // is a power of two + assert(bucket_count() >= HT_MIN_BUCKETS); + bool retval = false; + + // If you construct a hashtable with < HT_DEFAULT_STARTING_BUCKETS, + // we'll never shrink until you get relatively big, and we'll never + // shrink below HT_DEFAULT_STARTING_BUCKETS. Otherwise, something + // like "dense_hash_set<int> x; x.insert(4); x.erase(4);" will + // shrink us down to HT_MIN_BUCKETS buckets, which is too small. + const size_type num_remain = table.num_nonempty() - num_deleted; + const size_type shrink_threshold = settings.shrink_threshold(); + if (shrink_threshold > 0 && num_remain < shrink_threshold && + bucket_count() > HT_DEFAULT_STARTING_BUCKETS) { + const float shrink_factor = settings.shrink_factor(); + size_type sz = bucket_count() / 2; // find how much we should shrink + while (sz > HT_DEFAULT_STARTING_BUCKETS && + num_remain < static_cast<size_type>(sz * shrink_factor)) { + sz /= 2; // stay a power of 2 + } + sparse_hashtable tmp(MoveDontCopy, *this, sz); + swap(tmp); // now we are tmp + retval = true; + } + settings.set_consider_shrink(false); // because we just considered it + return retval; + } + + // We'll let you resize a hashtable -- though this makes us copy all! + // When you resize, you say, "make it big enough for this many more elements" + // Returns true if we actually resized, false if size was already ok. + bool resize_delta(size_type delta) { + bool did_resize = false; + if ( settings.consider_shrink() ) { // see if lots of deletes happened + if ( maybe_shrink() ) + did_resize = true; + } + if (table.num_nonempty() >= + (std::numeric_limits<size_type>::max)() - delta) { + throw std::length_error("resize overflow"); + } + if ( bucket_count() >= HT_MIN_BUCKETS && + (table.num_nonempty() + delta) <= settings.enlarge_threshold() ) + return did_resize; // we're ok as we are + + // Sometimes, we need to resize just to get rid of all the + // "deleted" buckets that are clogging up the hashtable. So when + // deciding whether to resize, count the deleted buckets (which + // are currently taking up room). But later, when we decide what + // size to resize to, *don't* count deleted buckets, since they + // get discarded during the resize. + const size_type needed_size = + settings.min_buckets(table.num_nonempty() + delta, 0); + if ( needed_size <= bucket_count() ) // we have enough buckets + return did_resize; + + size_type resize_to = + settings.min_buckets(table.num_nonempty() - num_deleted + delta, + bucket_count()); + if (resize_to < needed_size && // may double resize_to + resize_to < (std::numeric_limits<size_type>::max)() / 2) { + // This situation means that we have enough deleted elements, + // that once we purge them, we won't actually have needed to + // grow. But we may want to grow anyway: if we just purge one + // element, say, we'll have to grow anyway next time we + // insert. Might as well grow now, since we're already going + // through the trouble of copying (in order to purge the + // deleted elements). + const size_type target = + static_cast<size_type>(settings.shrink_size(resize_to*2)); + if (table.num_nonempty() - num_deleted + delta >= target) { + // Good, we won't be below the shrink threshhold even if we double. + resize_to *= 2; + } + } + + sparse_hashtable tmp(MoveDontCopy, *this, resize_to); + swap(tmp); // now we are tmp + return true; + } + + // Used to actually do the rehashing when we grow/shrink a hashtable + void copy_from(const sparse_hashtable &ht, size_type min_buckets_wanted) { + clear(); // clear table, set num_deleted to 0 + + // If we need to change the size of our table, do it now + const size_type resize_to = + settings.min_buckets(ht.size(), min_buckets_wanted); + if ( resize_to > bucket_count() ) { // we don't have enough buckets + table.resize(resize_to); // sets the number of buckets + settings.reset_thresholds(bucket_count()); + } + + // We use a normal iterator to get non-deleted bcks from ht + // We could use insert() here, but since we know there are + // no duplicates and no deleted items, we can be more efficient + assert((bucket_count() & (bucket_count()-1)) == 0); // a power of two + for ( const_iterator it = ht.begin(); it != ht.end(); ++it ) { + size_type num_probes = 0; // how many times we've probed + size_type bucknum; + const size_type bucket_count_minus_one = bucket_count() - 1; + for (bucknum = hash(get_key(*it)) & bucket_count_minus_one; + table.test(bucknum); // not empty + bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one) { + ++num_probes; + assert(num_probes < bucket_count() + && "Hashtable is full: an error in key_equal<> or hash<>"); + } + table.set(bucknum, *it); // copies the value to here + } + settings.inc_num_ht_copies(); + } + + // Implementation is like copy_from, but it destroys the table of the + // "from" guy by freeing sparsetable memory as we iterate. This is + // useful in resizing, since we're throwing away the "from" guy anyway. + void move_from(MoveDontCopyT mover, sparse_hashtable &ht, + size_type min_buckets_wanted) { + clear(); // clear table, set num_deleted to 0 + + // If we need to change the size of our table, do it now + size_type resize_to; + if ( mover == MoveDontGrow ) + resize_to = ht.bucket_count(); // keep same size as old ht + else // MoveDontCopy + resize_to = settings.min_buckets(ht.size(), min_buckets_wanted); + if ( resize_to > bucket_count() ) { // we don't have enough buckets + table.resize(resize_to); // sets the number of buckets + settings.reset_thresholds(bucket_count()); + } + + // We use a normal iterator to get non-deleted bcks from ht + // We could use insert() here, but since we know there are + // no duplicates and no deleted items, we can be more efficient + assert( (bucket_count() & (bucket_count()-1)) == 0); // a power of two + // THIS IS THE MAJOR LINE THAT DIFFERS FROM COPY_FROM(): + for ( destructive_iterator it = ht.destructive_begin(); + it != ht.destructive_end(); ++it ) { + size_type num_probes = 0; // how many times we've probed + size_type bucknum; + for ( bucknum = hash(get_key(*it)) & (bucket_count()-1); // h % buck_cnt + table.test(bucknum); // not empty + bucknum = (bucknum + JUMP_(key, num_probes)) & (bucket_count()-1) ) { + ++num_probes; + assert(num_probes < bucket_count() + && "Hashtable is full: an error in key_equal<> or hash<>"); + } + table.set(bucknum, *it); // copies the value to here + } + settings.inc_num_ht_copies(); + } + + + // Required by the spec for hashed associative container + public: + // Though the docs say this should be num_buckets, I think it's much + // more useful as num_elements. As a special feature, calling with + // req_elements==0 will cause us to shrink if we can, saving space. + void resize(size_type req_elements) { // resize to this or larger + if ( settings.consider_shrink() || req_elements == 0 ) + maybe_shrink(); + if ( req_elements > table.num_nonempty() ) // we only grow + resize_delta(req_elements - table.num_nonempty()); + } + + // Get and change the value of shrink_factor and enlarge_factor. The + // description at the beginning of this file explains how to choose + // the values. Setting the shrink parameter to 0.0 ensures that the + // table never shrinks. + void get_resizing_parameters(float* shrink, float* grow) const { + *shrink = settings.shrink_factor(); + *grow = settings.enlarge_factor(); + } + void set_resizing_parameters(float shrink, float grow) { + settings.set_resizing_parameters(shrink, grow); + settings.reset_thresholds(bucket_count()); + } + + // CONSTRUCTORS -- as required by the specs, we take a size, + // but also let you specify a hashfunction, key comparator, + // and key extractor. We also define a copy constructor and =. + // DESTRUCTOR -- the default is fine, surprisingly. + explicit sparse_hashtable(size_type expected_max_items_in_table = 0, + const HashFcn& hf = HashFcn(), + const EqualKey& eql = EqualKey(), + const ExtractKey& ext = ExtractKey(), + const SetKey& set = SetKey(), + const Alloc& alloc = Alloc()) + : settings(hf), + key_info(ext, set, eql), + num_deleted(0), + table((expected_max_items_in_table == 0 + ? HT_DEFAULT_STARTING_BUCKETS + : settings.min_buckets(expected_max_items_in_table, 0)), + alloc) { + settings.reset_thresholds(bucket_count()); + } + + // As a convenience for resize(), we allow an optional second argument + // which lets you make this new hashtable a different size than ht. + // We also provide a mechanism of saying you want to "move" the ht argument + // into us instead of copying. + sparse_hashtable(const sparse_hashtable& ht, + size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS) + : settings(ht.settings), + key_info(ht.key_info), + num_deleted(0), + table(0, ht.get_allocator()) { + settings.reset_thresholds(bucket_count()); + copy_from(ht, min_buckets_wanted); // copy_from() ignores deleted entries + } + sparse_hashtable(MoveDontCopyT mover, sparse_hashtable& ht, + size_type min_buckets_wanted = HT_DEFAULT_STARTING_BUCKETS) + : settings(ht.settings), + key_info(ht.key_info), + num_deleted(0), + table(0, ht.get_allocator()) { + settings.reset_thresholds(bucket_count()); + move_from(mover, ht, min_buckets_wanted); // ignores deleted entries + } + + sparse_hashtable& operator= (const sparse_hashtable& ht) { + if (&ht == this) return *this; // don't copy onto ourselves + settings = ht.settings; + key_info = ht.key_info; + num_deleted = ht.num_deleted; + // copy_from() calls clear and sets num_deleted to 0 too + copy_from(ht, HT_MIN_BUCKETS); + // we purposefully don't copy the allocator, which may not be copyable + return *this; + } + + // Many STL algorithms use swap instead of copy constructors + void swap(sparse_hashtable& ht) { + std::swap(settings, ht.settings); + std::swap(key_info, ht.key_info); + std::swap(num_deleted, ht.num_deleted); + table.swap(ht.table); + settings.reset_thresholds(bucket_count()); // also resets consider_shrink + ht.settings.reset_thresholds(ht.bucket_count()); + // we purposefully don't swap the allocator, which may not be swap-able + } + + // It's always nice to be able to clear a table without deallocating it + void clear() { + if (!empty() || (num_deleted != 0)) { + table.clear(); + } + settings.reset_thresholds(bucket_count()); + num_deleted = 0; + } + + // LOOKUP ROUTINES + private: + // Returns a pair of positions: 1st where the object is, 2nd where + // it would go if you wanted to insert it. 1st is ILLEGAL_BUCKET + // if object is not found; 2nd is ILLEGAL_BUCKET if it is. + // Note: because of deletions where-to-insert is not trivial: it's the + // first deleted bucket we see, as long as we don't find the key later + std::pair<size_type, size_type> find_position(const key_type &key) const { + size_type num_probes = 0; // how many times we've probed + const size_type bucket_count_minus_one = bucket_count() - 1; + size_type bucknum = hash(key) & bucket_count_minus_one; + size_type insert_pos = ILLEGAL_BUCKET; // where we would insert + SPARSEHASH_STAT_UPDATE(total_lookups += 1); + while ( 1 ) { // probe until something happens + if ( !table.test(bucknum) ) { // bucket is empty + SPARSEHASH_STAT_UPDATE(total_probes += num_probes); + if ( insert_pos == ILLEGAL_BUCKET ) // found no prior place to insert + return std::pair<size_type,size_type>(ILLEGAL_BUCKET, bucknum); + else + return std::pair<size_type,size_type>(ILLEGAL_BUCKET, insert_pos); + + } else if ( test_deleted(bucknum) ) {// keep searching, but mark to insert + if ( insert_pos == ILLEGAL_BUCKET ) + insert_pos = bucknum; + + } else if ( equals(key, get_key(table.unsafe_get(bucknum))) ) { + SPARSEHASH_STAT_UPDATE(total_probes += num_probes); + return std::pair<size_type,size_type>(bucknum, ILLEGAL_BUCKET); + } + ++num_probes; // we're doing another probe + bucknum = (bucknum + JUMP_(key, num_probes)) & bucket_count_minus_one; + assert(num_probes < bucket_count() + && "Hashtable is full: an error in key_equal<> or hash<>"); + } + } + + public: + + iterator find(const key_type& key) { + if ( size() == 0 ) return end(); + std::pair<size_type, size_type> pos = find_position(key); + if ( pos.first == ILLEGAL_BUCKET ) // alas, not there + return end(); + else + return iterator(this, table.get_iter(pos.first), table.nonempty_end()); + } + + const_iterator find(const key_type& key) const { + if ( size() == 0 ) return end(); + std::pair<size_type, size_type> pos = find_position(key); + if ( pos.first == ILLEGAL_BUCKET ) // alas, not there + return end(); + else + return const_iterator(this, + table.get_iter(pos.first), table.nonempty_end()); + } + + // This is a tr1 method: the bucket a given key is in, or what bucket + // it would be put in, if it were to be inserted. Shrug. + size_type bucket(const key_type& key) const { + std::pair<size_type, size_type> pos = find_position(key); + return pos.first == ILLEGAL_BUCKET ? pos.second : pos.first; + } + + // Counts how many elements have key key. For maps, it's either 0 or 1. + size_type count(const key_type &key) const { + std::pair<size_type, size_type> pos = find_position(key); + return pos.first == ILLEGAL_BUCKET ? 0 : 1; + } + + // Likewise, equal_range doesn't really make sense for us. Oh well. + std::pair<iterator,iterator> equal_range(const key_type& key) { + iterator pos = find(key); // either an iterator or end + if (pos == end()) { + return std::pair<iterator,iterator>(pos, pos); + } else { + const iterator startpos = pos++; + return std::pair<iterator,iterator>(startpos, pos); + } + } + std::pair<const_iterator,const_iterator> equal_range(const key_type& key) + const { + const_iterator pos = find(key); // either an iterator or end + if (pos == end()) { + return std::pair<const_iterator,const_iterator>(pos, pos); + } else { + const const_iterator startpos = pos++; + return std::pair<const_iterator,const_iterator>(startpos, pos); + } + } + + + // INSERTION ROUTINES + private: + // Private method used by insert_noresize and find_or_insert. + iterator insert_at(const_reference obj, size_type pos) { + if (size() >= max_size()) { + throw std::length_error("insert overflow"); + } + if ( test_deleted(pos) ) { // just replace if it's been deleted + // The set() below will undelete this object. We just worry about stats + assert(num_deleted > 0); + --num_deleted; // used to be, now it isn't + } + table.set(pos, obj); + return iterator(this, table.get_iter(pos), table.nonempty_end()); + } + + // If you know *this is big enough to hold obj, use this routine + std::pair<iterator, bool> insert_noresize(const_reference obj) { + // First, double-check we're not inserting delkey + assert((!settings.use_deleted() || !equals(get_key(obj), key_info.delkey)) + && "Inserting the deleted key"); + const std::pair<size_type,size_type> pos = find_position(get_key(obj)); + if ( pos.first != ILLEGAL_BUCKET) { // object was already there + return std::pair<iterator,bool>(iterator(this, table.get_iter(pos.first), + table.nonempty_end()), + false); // false: we didn't insert + } else { // pos.second says where to put it + return std::pair<iterator,bool>(insert_at(obj, pos.second), true); + } + } + + // Specializations of insert(it, it) depending on the power of the iterator: + // (1) Iterator supports operator-, resize before inserting + template <class ForwardIterator> + void insert(ForwardIterator f, ForwardIterator l, std::forward_iterator_tag) { + size_t dist = std::distance(f, l); + if (dist >= (std::numeric_limits<size_type>::max)()) { + throw std::length_error("insert-range overflow"); + } + resize_delta(static_cast<size_type>(dist)); + for ( ; dist > 0; --dist, ++f) { + insert_noresize(*f); + } + } + + // (2) Arbitrary iterator, can't tell how much to resize + template <class InputIterator> + void insert(InputIterator f, InputIterator l, std::input_iterator_tag) { + for ( ; f != l; ++f) + insert(*f); + } + + public: + // This is the normal insert routine, used by the outside world + std::pair<iterator, bool> insert(const_reference obj) { + resize_delta(1); // adding an object, grow if need be + return insert_noresize(obj); + } + + // When inserting a lot at a time, we specialize on the type of iterator + template <class InputIterator> + void insert(InputIterator f, InputIterator l) { + // specializes on iterator type + insert(f, l, + typename std::iterator_traits<InputIterator>::iterator_category()); + } + + // DefaultValue is a functor that takes a key and returns a value_type + // representing the default value to be inserted if none is found. + template <class DefaultValue> + value_type& find_or_insert(const key_type& key) { + // First, double-check we're not inserting delkey + assert((!settings.use_deleted() || !equals(key, key_info.delkey)) + && "Inserting the deleted key"); + const std::pair<size_type,size_type> pos = find_position(key); + DefaultValue default_value; + if ( pos.first != ILLEGAL_BUCKET) { // object was already there + return *table.get_iter(pos.first); + } else if (resize_delta(1)) { // needed to rehash to make room + // Since we resized, we can't use pos, so recalculate where to insert. + return *insert_noresize(default_value(key)).first; + } else { // no need to rehash, insert right here + return *insert_at(default_value(key), pos.second); + } + } + + // DELETION ROUTINES + size_type erase(const key_type& key) { + // First, double-check we're not erasing delkey. + assert((!settings.use_deleted() || !equals(key, key_info.delkey)) + && "Erasing the deleted key"); + assert(!settings.use_deleted() || !equals(key, key_info.delkey)); + const_iterator pos = find(key); // shrug: shouldn't need to be const + if ( pos != end() ) { + assert(!test_deleted(pos)); // or find() shouldn't have returned it + set_deleted(pos); + ++num_deleted; + // will think about shrink after next insert + settings.set_consider_shrink(true); + return 1; // because we deleted one thing + } else { + return 0; // because we deleted nothing + } + } + + // We return the iterator past the deleted item. + void erase(iterator pos) { + if ( pos == end() ) return; // sanity check + if ( set_deleted(pos) ) { // true if object has been newly deleted + ++num_deleted; + // will think about shrink after next insert + settings.set_consider_shrink(true); + } + } + + void erase(iterator f, iterator l) { + for ( ; f != l; ++f) { + if ( set_deleted(f) ) // should always be true + ++num_deleted; + } + // will think about shrink after next insert + settings.set_consider_shrink(true); + } + + // We allow you to erase a const_iterator just like we allow you to + // erase an iterator. This is in parallel to 'delete': you can delete + // a const pointer just like a non-const pointer. The logic is that + // you can't use the object after it's erased anyway, so it doesn't matter + // if it's const or not. + void erase(const_iterator pos) { + if ( pos == end() ) return; // sanity check + if ( set_deleted(pos) ) { // true if object has been newly deleted + ++num_deleted; + // will think about shrink after next insert + settings.set_consider_shrink(true); + } + } + void erase(const_iterator f, const_iterator l) { + for ( ; f != l; ++f) { + if ( set_deleted(f) ) // should always be true + ++num_deleted; + } + // will think about shrink after next insert + settings.set_consider_shrink(true); + } + + + // COMPARISON + bool operator==(const sparse_hashtable& ht) const { + if (size() != ht.size()) { + return false; + } else if (this == &ht) { + return true; + } else { + // Iterate through the elements in "this" and see if the + // corresponding element is in ht + for ( const_iterator it = begin(); it != end(); ++it ) { + const_iterator it2 = ht.find(get_key(*it)); + if ((it2 == ht.end()) || (*it != *it2)) { + return false; + } + } + return true; + } + } + bool operator!=(const sparse_hashtable& ht) const { + return !(*this == ht); + } + + + // I/O + // We support reading and writing hashtables to disk. NOTE that + // this only stores the hashtable metadata, not the stuff you've + // actually put in the hashtable! Alas, since I don't know how to + // write a hasher or key_equal, you have to make sure everything + // but the table is the same. We compact before writing. + // + // The OUTPUT type needs to support a Write() operation. File and + // OutputBuffer are appropriate types to pass in. + // + // The INPUT type needs to support a Read() operation. File and + // InputBuffer are appropriate types to pass in. + template <typename OUTPUT> + bool write_metadata(OUTPUT *fp) { + squash_deleted(); // so we don't have to worry about delkey + return table.write_metadata(fp); + } + + template <typename INPUT> + bool read_metadata(INPUT *fp) { + num_deleted = 0; // since we got rid before writing + const bool result = table.read_metadata(fp); + settings.reset_thresholds(bucket_count()); + return result; + } + + // Only meaningful if value_type is a POD. + template <typename OUTPUT> + bool write_nopointer_data(OUTPUT *fp) { + return table.write_nopointer_data(fp); + } + + // Only meaningful if value_type is a POD. + template <typename INPUT> + bool read_nopointer_data(INPUT *fp) { + return table.read_nopointer_data(fp); + } + + // INPUT and OUTPUT must be either a FILE, *or* a C++ stream + // (istream, ostream, etc) *or* a class providing + // Read(void*, size_t) and Write(const void*, size_t) + // (respectively), which writes a buffer into a stream + // (which the INPUT/OUTPUT instance presumably owns). + + typedef sparsehash_internal::pod_serializer<value_type> NopointerSerializer; + + // ValueSerializer: a functor. operator()(OUTPUT*, const value_type&) + template <typename ValueSerializer, typename OUTPUT> + bool serialize(ValueSerializer serializer, OUTPUT *fp) { + squash_deleted(); // so we don't have to worry about delkey + return table.serialize(serializer, fp); + } + + // ValueSerializer: a functor. operator()(INPUT*, value_type*) + template <typename ValueSerializer, typename INPUT> + bool unserialize(ValueSerializer serializer, INPUT *fp) { + num_deleted = 0; // since we got rid before writing + const bool result = table.unserialize(serializer, fp); + settings.reset_thresholds(bucket_count()); + return result; + } + + private: + // Table is the main storage class. + typedef sparsetable<value_type, DEFAULT_GROUP_SIZE, value_alloc_type> Table; + + // Package templated functors with the other types to eliminate memory + // needed for storing these zero-size operators. Since ExtractKey and + // hasher's operator() might have the same function signature, they + // must be packaged in different classes. + struct Settings : + sparsehash_internal::sh_hashtable_settings<key_type, hasher, + size_type, HT_MIN_BUCKETS> { + explicit Settings(const hasher& hf) + : sparsehash_internal::sh_hashtable_settings<key_type, hasher, + size_type, HT_MIN_BUCKETS>( + hf, HT_OCCUPANCY_PCT / 100.0f, HT_EMPTY_PCT / 100.0f) {} + }; + + // KeyInfo stores delete key and packages zero-size functors: + // ExtractKey and SetKey. + class KeyInfo : public ExtractKey, public SetKey, public EqualKey { + public: + KeyInfo(const ExtractKey& ek, const SetKey& sk, const EqualKey& eq) + : ExtractKey(ek), + SetKey(sk), + EqualKey(eq) { + } + // We want to return the exact same type as ExtractKey: Key or const Key& + typename ExtractKey::result_type get_key(const_reference v) const { + return ExtractKey::operator()(v); + } + void set_key(pointer v, const key_type& k) const { + SetKey::operator()(v, k); + } + bool equals(const key_type& a, const key_type& b) const { + return EqualKey::operator()(a, b); + } + + // Which key marks deleted entries. + // TODO(csilvers): make a pointer, and get rid of use_deleted (benchmark!) + typename base::remove_const<key_type>::type delkey; + }; + + // Utility functions to access the templated operators + size_type hash(const key_type& v) const { + return settings.hash(v); + } + bool equals(const key_type& a, const key_type& b) const { + return key_info.equals(a, b); + } + typename ExtractKey::result_type get_key(const_reference v) const { + return key_info.get_key(v); + } + void set_key(pointer v, const key_type& k) const { + key_info.set_key(v, k); + } + + private: + // Actual data + Settings settings; + KeyInfo key_info; + size_type num_deleted; // how many occupied buckets are marked deleted + Table table; // holds num_buckets and num_elements too +}; + + +// We need a global swap as well +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +inline void swap(sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> &x, + sparse_hashtable<V,K,HF,ExK,SetK,EqK,A> &y) { + x.swap(y); +} + +#undef JUMP_ + +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +const typename sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::size_type + sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::ILLEGAL_BUCKET; + +// How full we let the table get before we resize. Knuth says .8 is +// good -- higher causes us to probe too much, though saves memory +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +const int sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT = 80; + +// How empty we let the table get before we resize lower. +// It should be less than OCCUPANCY_PCT / 2 or we thrash resizing +template <class V, class K, class HF, class ExK, class SetK, class EqK, class A> +const int sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_EMPTY_PCT + = static_cast<int>(0.4 * + sparse_hashtable<V,K,HF,ExK,SetK,EqK,A>::HT_OCCUPANCY_PCT); + +_END_GOOGLE_NAMESPACE_ + +#endif /* _SPARSEHASHTABLE_H_ */ diff --git a/contrib/libs/sparsehash/src/sparsehash/sparse_hash_map b/contrib/libs/sparsehash/src/sparsehash/sparse_hash_map index 2520deb505..1687a8b11c 100644 --- a/contrib/libs/sparsehash/src/sparsehash/sparse_hash_map +++ b/contrib/libs/sparsehash/src/sparsehash/sparse_hash_map @@ -1,363 +1,363 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// -// This is just a very thin wrapper over sparsehashtable.h, just -// like sgi stl's stl_hash_map is a very thin wrapper over -// stl_hashtable. The major thing we define is operator[], because -// we have a concept of a data_type which stl_hashtable doesn't -// (it only has a key and a value). -// -// We adhere mostly to the STL semantics for hash-map. One important -// exception is that insert() may invalidate iterators entirely -- STL -// semantics are that insert() may reorder iterators, but they all -// still refer to something valid in the hashtable. Not so for us. -// Likewise, insert() may invalidate pointers into the hashtable. -// (Whether insert invalidates iterators and pointers depends on -// whether it results in a hashtable resize). On the plus side, -// delete() doesn't invalidate iterators or pointers at all, or even -// change the ordering of elements. -// -// Here are a few "power user" tips: -// -// 1) set_deleted_key(): -// Unlike STL's hash_map, if you want to use erase() you -// *must* call set_deleted_key() after construction. -// -// 2) resize(0): -// When an item is deleted, its memory isn't freed right -// away. This is what allows you to iterate over a hashtable -// and call erase() without invalidating the iterator. -// To force the memory to be freed, call resize(0). -// For tr1 compatibility, this can also be called as rehash(0). -// -// 3) min_load_factor(0.0) -// Setting the minimum load factor to 0.0 guarantees that -// the hash table will never shrink. -// -// Roughly speaking: -// (1) dense_hash_map: fastest, uses the most memory unless entries are small -// (2) sparse_hash_map: slowest, uses the least memory -// (3) hash_map / unordered_map (STL): in the middle -// -// Typically I use sparse_hash_map when I care about space and/or when -// I need to save the hashtable on disk. I use hash_map otherwise. I -// don't personally use dense_hash_map ever; some people use it for -// small maps with lots of lookups. -// -// - dense_hash_map has, typically, about 78% memory overhead (if your -// data takes up X bytes, the hash_map uses .78X more bytes in overhead). -// - sparse_hash_map has about 4 bits overhead per entry. -// - sparse_hash_map can be 3-7 times slower than the others for lookup and, -// especially, inserts. See time_hash_map.cc for details. -// -// See /usr/(local/)?doc/sparsehash-*/sparse_hash_map.html -// for information about how to use this class. - -#ifndef _SPARSE_HASH_MAP_H_ -#define _SPARSE_HASH_MAP_H_ - -#include <sparsehash/internal/sparseconfig.h> -#include <algorithm> // needed by stl_alloc -#include <functional> // for equal_to<>, select1st<>, etc -#include <memory> // for alloc -#include <utility> // for pair<> -#include <sparsehash/internal/libc_allocator_with_realloc.h> -#include <sparsehash/internal/sparsehashtable.h> // IWYU pragma: export -#include HASH_FUN_H // for hash<> -_START_GOOGLE_NAMESPACE_ - -template <class Key, class T, - class HashFcn = SPARSEHASH_HASH<Key>, // defined in sparseconfig.h - class EqualKey = std::equal_to<Key>, - class Alloc = libc_allocator_with_realloc<std::pair<const Key, T> > > -class sparse_hash_map { - private: - // Apparently select1st is not stl-standard, so we define our own - struct SelectKey { - typedef const Key& result_type; - const Key& operator()(const std::pair<const Key, T>& p) const { - return p.first; - } - }; - struct SetKey { - void operator()(std::pair<const Key, T>* value, const Key& new_key) const { - *const_cast<Key*>(&value->first) = new_key; - // It would be nice to clear the rest of value here as well, in - // case it's taking up a lot of memory. We do this by clearing - // the value. This assumes T has a zero-arg constructor! - value->second = T(); - } - }; - // For operator[]. - struct DefaultValue { - std::pair<const Key, T> operator()(const Key& key) { - return std::make_pair(key, T()); - } - }; - - // The actual data - typedef sparse_hashtable<std::pair<const Key, T>, Key, HashFcn, SelectKey, - SetKey, EqualKey, Alloc> ht; - ht rep; - - public: - typedef typename ht::key_type key_type; - typedef T data_type; - typedef T mapped_type; - typedef typename ht::value_type value_type; - typedef typename ht::hasher hasher; - typedef typename ht::key_equal key_equal; - typedef Alloc allocator_type; - - typedef typename ht::size_type size_type; - typedef typename ht::difference_type difference_type; - typedef typename ht::pointer pointer; - typedef typename ht::const_pointer const_pointer; - typedef typename ht::reference reference; - typedef typename ht::const_reference const_reference; - - typedef typename ht::iterator iterator; - typedef typename ht::const_iterator const_iterator; - typedef typename ht::local_iterator local_iterator; - typedef typename ht::const_local_iterator const_local_iterator; - - // Iterator functions - iterator begin() { return rep.begin(); } - iterator end() { return rep.end(); } - const_iterator begin() const { return rep.begin(); } - const_iterator end() const { return rep.end(); } - - // These come from tr1's unordered_map. For us, a bucket has 0 or 1 elements. - local_iterator begin(size_type i) { return rep.begin(i); } - local_iterator end(size_type i) { return rep.end(i); } - const_local_iterator begin(size_type i) const { return rep.begin(i); } - const_local_iterator end(size_type i) const { return rep.end(i); } - - // Accessor functions - allocator_type get_allocator() const { return rep.get_allocator(); } - hasher hash_funct() const { return rep.hash_funct(); } - hasher hash_function() const { return hash_funct(); } - key_equal key_eq() const { return rep.key_eq(); } - - - // Constructors - explicit sparse_hash_map(size_type expected_max_items_in_table = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { - } - - template <class InputIterator> - sparse_hash_map(InputIterator f, InputIterator l, - size_type expected_max_items_in_table = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { - rep.insert(f, l); - } - // We use the default copy constructor - // We use the default operator=() - // We use the default destructor - - void clear() { rep.clear(); } - void swap(sparse_hash_map& hs) { rep.swap(hs.rep); } - - - // Functions concerning size - size_type size() const { return rep.size(); } - size_type max_size() const { return rep.max_size(); } - bool empty() const { return rep.empty(); } - size_type bucket_count() const { return rep.bucket_count(); } - size_type max_bucket_count() const { return rep.max_bucket_count(); } - - // These are tr1 methods. bucket() is the bucket the key is or would be in. - size_type bucket_size(size_type i) const { return rep.bucket_size(i); } - size_type bucket(const key_type& key) const { return rep.bucket(key); } - float load_factor() const { - return size() * 1.0f / bucket_count(); - } - float max_load_factor() const { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - return grow; - } - void max_load_factor(float new_grow) { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - rep.set_resizing_parameters(shrink, new_grow); - } - // These aren't tr1 methods but perhaps ought to be. - float min_load_factor() const { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - return shrink; - } - void min_load_factor(float new_shrink) { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - rep.set_resizing_parameters(new_shrink, grow); - } - // Deprecated; use min_load_factor() or max_load_factor() instead. - void set_resizing_parameters(float shrink, float grow) { - rep.set_resizing_parameters(shrink, grow); - } - - void resize(size_type hint) { rep.resize(hint); } - void rehash(size_type hint) { resize(hint); } // the tr1 name - - // Lookup routines - iterator find(const key_type& key) { return rep.find(key); } - const_iterator find(const key_type& key) const { return rep.find(key); } - - data_type& operator[](const key_type& key) { // This is our value-add! - // If key is in the hashtable, returns find(key)->second, - // otherwise returns insert(value_type(key, T()).first->second. - // Note it does not create an empty T unless the find fails. - return rep.template find_or_insert<DefaultValue>(key).second; - } - - size_type count(const key_type& key) const { return rep.count(key); } - - std::pair<iterator, iterator> equal_range(const key_type& key) { - return rep.equal_range(key); - } - std::pair<const_iterator, const_iterator> equal_range(const key_type& key) - const { - return rep.equal_range(key); - } - - // Insertion routines - std::pair<iterator, bool> insert(const value_type& obj) { - return rep.insert(obj); - } - template <class InputIterator> void insert(InputIterator f, InputIterator l) { - rep.insert(f, l); - } - void insert(const_iterator f, const_iterator l) { - rep.insert(f, l); - } - // Required for std::insert_iterator; the passed-in iterator is ignored. - iterator insert(iterator, const value_type& obj) { - return insert(obj).first; - } - - // Deletion routines - // THESE ARE NON-STANDARD! I make you specify an "impossible" key - // value to identify deleted buckets. You can change the key as - // time goes on, or get rid of it entirely to be insert-only. - void set_deleted_key(const key_type& key) { - rep.set_deleted_key(key); - } - void clear_deleted_key() { rep.clear_deleted_key(); } - key_type deleted_key() const { return rep.deleted_key(); } - - // These are standard - size_type erase(const key_type& key) { return rep.erase(key); } - void erase(iterator it) { rep.erase(it); } - void erase(iterator f, iterator l) { rep.erase(f, l); } - - - // Comparison - bool operator==(const sparse_hash_map& hs) const { return rep == hs.rep; } - bool operator!=(const sparse_hash_map& hs) const { return rep != hs.rep; } - - - // I/O -- this is an add-on for writing metainformation to disk - // - // For maximum flexibility, this does not assume a particular - // file type (though it will probably be a FILE *). We just pass - // the fp through to rep. - - // If your keys and values are simple enough, you can pass this - // serializer to serialize()/unserialize(). "Simple enough" means - // value_type is a POD type that contains no pointers. Note, - // however, we don't try to normalize endianness. - typedef typename ht::NopointerSerializer NopointerSerializer; - - // serializer: a class providing operator()(OUTPUT*, const value_type&) - // (writing value_type to OUTPUT). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a - // pointer to a class providing size_t Write(const void*, size_t), - // which writes a buffer into a stream (which fp presumably - // owns) and returns the number of bytes successfully written. - // Note basic_ostream<not_char> is not currently supported. - template <typename ValueSerializer, typename OUTPUT> - bool serialize(ValueSerializer serializer, OUTPUT* fp) { - return rep.serialize(serializer, fp); - } - - // serializer: a functor providing operator()(INPUT*, value_type*) - // (reading from INPUT and into value_type). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a - // pointer to a class providing size_t Read(void*, size_t), - // which reads into a buffer from a stream (which fp presumably - // owns) and returns the number of bytes successfully read. - // Note basic_istream<not_char> is not currently supported. - // NOTE: Since value_type is std::pair<const Key, T>, ValueSerializer - // may need to do a const cast in order to fill in the key. - // NOTE: if Key or T are not POD types, the serializer MUST use - // placement-new to initialize their values, rather than a normal - // equals-assignment or similar. (The value_type* passed into the - // serializer points to garbage memory.) - template <typename ValueSerializer, typename INPUT> - bool unserialize(ValueSerializer serializer, INPUT* fp) { - return rep.unserialize(serializer, fp); - } - - // The four methods below are DEPRECATED. - // Use serialize() and unserialize() for new code. - template <typename OUTPUT> - bool write_metadata(OUTPUT *fp) { return rep.write_metadata(fp); } - - template <typename INPUT> - bool read_metadata(INPUT *fp) { return rep.read_metadata(fp); } - - template <typename OUTPUT> - bool write_nopointer_data(OUTPUT *fp) { return rep.write_nopointer_data(fp); } - - template <typename INPUT> - bool read_nopointer_data(INPUT *fp) { return rep.read_nopointer_data(fp); } -}; - -// We need a global swap as well -template <class Key, class T, class HashFcn, class EqualKey, class Alloc> -inline void swap(sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm1, - sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm2) { - hm1.swap(hm2); -} - -_END_GOOGLE_NAMESPACE_ - -#endif /* _SPARSE_HASH_MAP_H_ */ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// +// This is just a very thin wrapper over sparsehashtable.h, just +// like sgi stl's stl_hash_map is a very thin wrapper over +// stl_hashtable. The major thing we define is operator[], because +// we have a concept of a data_type which stl_hashtable doesn't +// (it only has a key and a value). +// +// We adhere mostly to the STL semantics for hash-map. One important +// exception is that insert() may invalidate iterators entirely -- STL +// semantics are that insert() may reorder iterators, but they all +// still refer to something valid in the hashtable. Not so for us. +// Likewise, insert() may invalidate pointers into the hashtable. +// (Whether insert invalidates iterators and pointers depends on +// whether it results in a hashtable resize). On the plus side, +// delete() doesn't invalidate iterators or pointers at all, or even +// change the ordering of elements. +// +// Here are a few "power user" tips: +// +// 1) set_deleted_key(): +// Unlike STL's hash_map, if you want to use erase() you +// *must* call set_deleted_key() after construction. +// +// 2) resize(0): +// When an item is deleted, its memory isn't freed right +// away. This is what allows you to iterate over a hashtable +// and call erase() without invalidating the iterator. +// To force the memory to be freed, call resize(0). +// For tr1 compatibility, this can also be called as rehash(0). +// +// 3) min_load_factor(0.0) +// Setting the minimum load factor to 0.0 guarantees that +// the hash table will never shrink. +// +// Roughly speaking: +// (1) dense_hash_map: fastest, uses the most memory unless entries are small +// (2) sparse_hash_map: slowest, uses the least memory +// (3) hash_map / unordered_map (STL): in the middle +// +// Typically I use sparse_hash_map when I care about space and/or when +// I need to save the hashtable on disk. I use hash_map otherwise. I +// don't personally use dense_hash_map ever; some people use it for +// small maps with lots of lookups. +// +// - dense_hash_map has, typically, about 78% memory overhead (if your +// data takes up X bytes, the hash_map uses .78X more bytes in overhead). +// - sparse_hash_map has about 4 bits overhead per entry. +// - sparse_hash_map can be 3-7 times slower than the others for lookup and, +// especially, inserts. See time_hash_map.cc for details. +// +// See /usr/(local/)?doc/sparsehash-*/sparse_hash_map.html +// for information about how to use this class. + +#ifndef _SPARSE_HASH_MAP_H_ +#define _SPARSE_HASH_MAP_H_ + +#include <sparsehash/internal/sparseconfig.h> +#include <algorithm> // needed by stl_alloc +#include <functional> // for equal_to<>, select1st<>, etc +#include <memory> // for alloc +#include <utility> // for pair<> +#include <sparsehash/internal/libc_allocator_with_realloc.h> +#include <sparsehash/internal/sparsehashtable.h> // IWYU pragma: export +#include HASH_FUN_H // for hash<> +_START_GOOGLE_NAMESPACE_ + +template <class Key, class T, + class HashFcn = SPARSEHASH_HASH<Key>, // defined in sparseconfig.h + class EqualKey = std::equal_to<Key>, + class Alloc = libc_allocator_with_realloc<std::pair<const Key, T> > > +class sparse_hash_map { + private: + // Apparently select1st is not stl-standard, so we define our own + struct SelectKey { + typedef const Key& result_type; + const Key& operator()(const std::pair<const Key, T>& p) const { + return p.first; + } + }; + struct SetKey { + void operator()(std::pair<const Key, T>* value, const Key& new_key) const { + *const_cast<Key*>(&value->first) = new_key; + // It would be nice to clear the rest of value here as well, in + // case it's taking up a lot of memory. We do this by clearing + // the value. This assumes T has a zero-arg constructor! + value->second = T(); + } + }; + // For operator[]. + struct DefaultValue { + std::pair<const Key, T> operator()(const Key& key) { + return std::make_pair(key, T()); + } + }; + + // The actual data + typedef sparse_hashtable<std::pair<const Key, T>, Key, HashFcn, SelectKey, + SetKey, EqualKey, Alloc> ht; + ht rep; + + public: + typedef typename ht::key_type key_type; + typedef T data_type; + typedef T mapped_type; + typedef typename ht::value_type value_type; + typedef typename ht::hasher hasher; + typedef typename ht::key_equal key_equal; + typedef Alloc allocator_type; + + typedef typename ht::size_type size_type; + typedef typename ht::difference_type difference_type; + typedef typename ht::pointer pointer; + typedef typename ht::const_pointer const_pointer; + typedef typename ht::reference reference; + typedef typename ht::const_reference const_reference; + + typedef typename ht::iterator iterator; + typedef typename ht::const_iterator const_iterator; + typedef typename ht::local_iterator local_iterator; + typedef typename ht::const_local_iterator const_local_iterator; + + // Iterator functions + iterator begin() { return rep.begin(); } + iterator end() { return rep.end(); } + const_iterator begin() const { return rep.begin(); } + const_iterator end() const { return rep.end(); } + + // These come from tr1's unordered_map. For us, a bucket has 0 or 1 elements. + local_iterator begin(size_type i) { return rep.begin(i); } + local_iterator end(size_type i) { return rep.end(i); } + const_local_iterator begin(size_type i) const { return rep.begin(i); } + const_local_iterator end(size_type i) const { return rep.end(i); } + + // Accessor functions + allocator_type get_allocator() const { return rep.get_allocator(); } + hasher hash_funct() const { return rep.hash_funct(); } + hasher hash_function() const { return hash_funct(); } + key_equal key_eq() const { return rep.key_eq(); } + + + // Constructors + explicit sparse_hash_map(size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { + } + + template <class InputIterator> + sparse_hash_map(InputIterator f, InputIterator l, + size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, SelectKey(), SetKey(), alloc) { + rep.insert(f, l); + } + // We use the default copy constructor + // We use the default operator=() + // We use the default destructor + + void clear() { rep.clear(); } + void swap(sparse_hash_map& hs) { rep.swap(hs.rep); } + + + // Functions concerning size + size_type size() const { return rep.size(); } + size_type max_size() const { return rep.max_size(); } + bool empty() const { return rep.empty(); } + size_type bucket_count() const { return rep.bucket_count(); } + size_type max_bucket_count() const { return rep.max_bucket_count(); } + + // These are tr1 methods. bucket() is the bucket the key is or would be in. + size_type bucket_size(size_type i) const { return rep.bucket_size(i); } + size_type bucket(const key_type& key) const { return rep.bucket(key); } + float load_factor() const { + return size() * 1.0f / bucket_count(); + } + float max_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return grow; + } + void max_load_factor(float new_grow) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(shrink, new_grow); + } + // These aren't tr1 methods but perhaps ought to be. + float min_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return shrink; + } + void min_load_factor(float new_shrink) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(new_shrink, grow); + } + // Deprecated; use min_load_factor() or max_load_factor() instead. + void set_resizing_parameters(float shrink, float grow) { + rep.set_resizing_parameters(shrink, grow); + } + + void resize(size_type hint) { rep.resize(hint); } + void rehash(size_type hint) { resize(hint); } // the tr1 name + + // Lookup routines + iterator find(const key_type& key) { return rep.find(key); } + const_iterator find(const key_type& key) const { return rep.find(key); } + + data_type& operator[](const key_type& key) { // This is our value-add! + // If key is in the hashtable, returns find(key)->second, + // otherwise returns insert(value_type(key, T()).first->second. + // Note it does not create an empty T unless the find fails. + return rep.template find_or_insert<DefaultValue>(key).second; + } + + size_type count(const key_type& key) const { return rep.count(key); } + + std::pair<iterator, iterator> equal_range(const key_type& key) { + return rep.equal_range(key); + } + std::pair<const_iterator, const_iterator> equal_range(const key_type& key) + const { + return rep.equal_range(key); + } + + // Insertion routines + std::pair<iterator, bool> insert(const value_type& obj) { + return rep.insert(obj); + } + template <class InputIterator> void insert(InputIterator f, InputIterator l) { + rep.insert(f, l); + } + void insert(const_iterator f, const_iterator l) { + rep.insert(f, l); + } + // Required for std::insert_iterator; the passed-in iterator is ignored. + iterator insert(iterator, const value_type& obj) { + return insert(obj).first; + } + + // Deletion routines + // THESE ARE NON-STANDARD! I make you specify an "impossible" key + // value to identify deleted buckets. You can change the key as + // time goes on, or get rid of it entirely to be insert-only. + void set_deleted_key(const key_type& key) { + rep.set_deleted_key(key); + } + void clear_deleted_key() { rep.clear_deleted_key(); } + key_type deleted_key() const { return rep.deleted_key(); } + + // These are standard + size_type erase(const key_type& key) { return rep.erase(key); } + void erase(iterator it) { rep.erase(it); } + void erase(iterator f, iterator l) { rep.erase(f, l); } + + + // Comparison + bool operator==(const sparse_hash_map& hs) const { return rep == hs.rep; } + bool operator!=(const sparse_hash_map& hs) const { return rep != hs.rep; } + + + // I/O -- this is an add-on for writing metainformation to disk + // + // For maximum flexibility, this does not assume a particular + // file type (though it will probably be a FILE *). We just pass + // the fp through to rep. + + // If your keys and values are simple enough, you can pass this + // serializer to serialize()/unserialize(). "Simple enough" means + // value_type is a POD type that contains no pointers. Note, + // however, we don't try to normalize endianness. + typedef typename ht::NopointerSerializer NopointerSerializer; + + // serializer: a class providing operator()(OUTPUT*, const value_type&) + // (writing value_type to OUTPUT). You can specify a + // NopointerSerializer object if appropriate (see above). + // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a + // pointer to a class providing size_t Write(const void*, size_t), + // which writes a buffer into a stream (which fp presumably + // owns) and returns the number of bytes successfully written. + // Note basic_ostream<not_char> is not currently supported. + template <typename ValueSerializer, typename OUTPUT> + bool serialize(ValueSerializer serializer, OUTPUT* fp) { + return rep.serialize(serializer, fp); + } + + // serializer: a functor providing operator()(INPUT*, value_type*) + // (reading from INPUT and into value_type). You can specify a + // NopointerSerializer object if appropriate (see above). + // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a + // pointer to a class providing size_t Read(void*, size_t), + // which reads into a buffer from a stream (which fp presumably + // owns) and returns the number of bytes successfully read. + // Note basic_istream<not_char> is not currently supported. + // NOTE: Since value_type is std::pair<const Key, T>, ValueSerializer + // may need to do a const cast in order to fill in the key. + // NOTE: if Key or T are not POD types, the serializer MUST use + // placement-new to initialize their values, rather than a normal + // equals-assignment or similar. (The value_type* passed into the + // serializer points to garbage memory.) + template <typename ValueSerializer, typename INPUT> + bool unserialize(ValueSerializer serializer, INPUT* fp) { + return rep.unserialize(serializer, fp); + } + + // The four methods below are DEPRECATED. + // Use serialize() and unserialize() for new code. + template <typename OUTPUT> + bool write_metadata(OUTPUT *fp) { return rep.write_metadata(fp); } + + template <typename INPUT> + bool read_metadata(INPUT *fp) { return rep.read_metadata(fp); } + + template <typename OUTPUT> + bool write_nopointer_data(OUTPUT *fp) { return rep.write_nopointer_data(fp); } + + template <typename INPUT> + bool read_nopointer_data(INPUT *fp) { return rep.read_nopointer_data(fp); } +}; + +// We need a global swap as well +template <class Key, class T, class HashFcn, class EqualKey, class Alloc> +inline void swap(sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm1, + sparse_hash_map<Key, T, HashFcn, EqualKey, Alloc>& hm2) { + hm1.swap(hm2); +} + +_END_GOOGLE_NAMESPACE_ + +#endif /* _SPARSE_HASH_MAP_H_ */ diff --git a/contrib/libs/sparsehash/src/sparsehash/sparse_hash_set b/contrib/libs/sparsehash/src/sparsehash/sparse_hash_set index 555c425514..ae4a97a62c 100644 --- a/contrib/libs/sparsehash/src/sparsehash/sparse_hash_set +++ b/contrib/libs/sparsehash/src/sparsehash/sparse_hash_set @@ -1,338 +1,338 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// -// This is just a very thin wrapper over sparsehashtable.h, just -// like sgi stl's stl_hash_set is a very thin wrapper over -// stl_hashtable. The major thing we define is operator[], because -// we have a concept of a data_type which stl_hashtable doesn't -// (it only has a key and a value). -// -// This is more different from sparse_hash_map than you might think, -// because all iterators for sets are const (you obviously can't -// change the key, and for sets there is no value). -// -// We adhere mostly to the STL semantics for hash-map. One important -// exception is that insert() may invalidate iterators entirely -- STL -// semantics are that insert() may reorder iterators, but they all -// still refer to something valid in the hashtable. Not so for us. -// Likewise, insert() may invalidate pointers into the hashtable. -// (Whether insert invalidates iterators and pointers depends on -// whether it results in a hashtable resize). On the plus side, -// delete() doesn't invalidate iterators or pointers at all, or even -// change the ordering of elements. -// -// Here are a few "power user" tips: -// -// 1) set_deleted_key(): -// Unlike STL's hash_map, if you want to use erase() you -// *must* call set_deleted_key() after construction. -// -// 2) resize(0): -// When an item is deleted, its memory isn't freed right -// away. This allows you to iterate over a hashtable, -// and call erase(), without invalidating the iterator. -// To force the memory to be freed, call resize(0). -// For tr1 compatibility, this can also be called as rehash(0). -// -// 3) min_load_factor(0.0) -// Setting the minimum load factor to 0.0 guarantees that -// the hash table will never shrink. -// -// Roughly speaking: -// (1) dense_hash_set: fastest, uses the most memory unless entries are small -// (2) sparse_hash_set: slowest, uses the least memory -// (3) hash_set / unordered_set (STL): in the middle -// -// Typically I use sparse_hash_set when I care about space and/or when -// I need to save the hashtable on disk. I use hash_set otherwise. I -// don't personally use dense_hash_set ever; some people use it for -// small sets with lots of lookups. -// -// - dense_hash_set has, typically, about 78% memory overhead (if your -// data takes up X bytes, the hash_set uses .78X more bytes in overhead). -// - sparse_hash_set has about 4 bits overhead per entry. -// - sparse_hash_set can be 3-7 times slower than the others for lookup and, -// especially, inserts. See time_hash_map.cc for details. -// -// See /usr/(local/)?doc/sparsehash-*/sparse_hash_set.html -// for information about how to use this class. - -#ifndef _SPARSE_HASH_SET_H_ -#define _SPARSE_HASH_SET_H_ - -#include <sparsehash/internal/sparseconfig.h> -#include <algorithm> // needed by stl_alloc -#include <functional> // for equal_to<> -#include <memory> // for alloc (which we don't use) -#include <utility> // for pair<> -#include <sparsehash/internal/libc_allocator_with_realloc.h> -#include <sparsehash/internal/sparsehashtable.h> // IWYU pragma: export -#include HASH_FUN_H // for hash<> - -_START_GOOGLE_NAMESPACE_ - -template <class Value, - class HashFcn = SPARSEHASH_HASH<Value>, // defined in sparseconfig.h - class EqualKey = std::equal_to<Value>, - class Alloc = libc_allocator_with_realloc<Value> > -class sparse_hash_set { - private: - // Apparently identity is not stl-standard, so we define our own - struct Identity { - typedef const Value& result_type; - const Value& operator()(const Value& v) const { return v; } - }; - struct SetKey { - void operator()(Value* value, const Value& new_key) const { - *value = new_key; - } - }; - - typedef sparse_hashtable<Value, Value, HashFcn, Identity, SetKey, - EqualKey, Alloc> ht; - ht rep; - - public: - typedef typename ht::key_type key_type; - typedef typename ht::value_type value_type; - typedef typename ht::hasher hasher; - typedef typename ht::key_equal key_equal; - typedef Alloc allocator_type; - - typedef typename ht::size_type size_type; - typedef typename ht::difference_type difference_type; - typedef typename ht::const_pointer pointer; - typedef typename ht::const_pointer const_pointer; - typedef typename ht::const_reference reference; - typedef typename ht::const_reference const_reference; - - typedef typename ht::const_iterator iterator; - typedef typename ht::const_iterator const_iterator; - typedef typename ht::const_local_iterator local_iterator; - typedef typename ht::const_local_iterator const_local_iterator; - - - // Iterator functions -- recall all iterators are const - iterator begin() const { return rep.begin(); } - iterator end() const { return rep.end(); } - - // These come from tr1's unordered_set. For us, a bucket has 0 or 1 elements. - local_iterator begin(size_type i) const { return rep.begin(i); } - local_iterator end(size_type i) const { return rep.end(i); } - - - // Accessor functions - allocator_type get_allocator() const { return rep.get_allocator(); } - hasher hash_funct() const { return rep.hash_funct(); } - hasher hash_function() const { return hash_funct(); } // tr1 name - key_equal key_eq() const { return rep.key_eq(); } - - - // Constructors - explicit sparse_hash_set(size_type expected_max_items_in_table = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { - } - - template <class InputIterator> - sparse_hash_set(InputIterator f, InputIterator l, - size_type expected_max_items_in_table = 0, - const hasher& hf = hasher(), - const key_equal& eql = key_equal(), - const allocator_type& alloc = allocator_type()) - : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { - rep.insert(f, l); - } - // We use the default copy constructor - // We use the default operator=() - // We use the default destructor - - void clear() { rep.clear(); } - void swap(sparse_hash_set& hs) { rep.swap(hs.rep); } - - - // Functions concerning size - size_type size() const { return rep.size(); } - size_type max_size() const { return rep.max_size(); } - bool empty() const { return rep.empty(); } - size_type bucket_count() const { return rep.bucket_count(); } - size_type max_bucket_count() const { return rep.max_bucket_count(); } - - // These are tr1 methods. bucket() is the bucket the key is or would be in. - size_type bucket_size(size_type i) const { return rep.bucket_size(i); } - size_type bucket(const key_type& key) const { return rep.bucket(key); } - float load_factor() const { - return size() * 1.0f / bucket_count(); - } - float max_load_factor() const { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - return grow; - } - void max_load_factor(float new_grow) { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - rep.set_resizing_parameters(shrink, new_grow); - } - // These aren't tr1 methods but perhaps ought to be. - float min_load_factor() const { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - return shrink; - } - void min_load_factor(float new_shrink) { - float shrink, grow; - rep.get_resizing_parameters(&shrink, &grow); - rep.set_resizing_parameters(new_shrink, grow); - } - // Deprecated; use min_load_factor() or max_load_factor() instead. - void set_resizing_parameters(float shrink, float grow) { - rep.set_resizing_parameters(shrink, grow); - } - - void resize(size_type hint) { rep.resize(hint); } - void rehash(size_type hint) { resize(hint); } // the tr1 name - - // Lookup routines - iterator find(const key_type& key) const { return rep.find(key); } - - size_type count(const key_type& key) const { return rep.count(key); } - - std::pair<iterator, iterator> equal_range(const key_type& key) const { - return rep.equal_range(key); - } - - - // Insertion routines - std::pair<iterator, bool> insert(const value_type& obj) { - std::pair<typename ht::iterator, bool> p = rep.insert(obj); - return std::pair<iterator, bool>(p.first, p.second); // const to non-const - } - template <class InputIterator> void insert(InputIterator f, InputIterator l) { - rep.insert(f, l); - } - void insert(const_iterator f, const_iterator l) { - rep.insert(f, l); - } - // Required for std::insert_iterator; the passed-in iterator is ignored. - iterator insert(iterator, const value_type& obj) { - return insert(obj).first; - } - - // Deletion routines - // THESE ARE NON-STANDARD! I make you specify an "impossible" key - // value to identify deleted buckets. You can change the key as - // time goes on, or get rid of it entirely to be insert-only. - void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); } - void clear_deleted_key() { rep.clear_deleted_key(); } - key_type deleted_key() const { return rep.deleted_key(); } - - // These are standard - size_type erase(const key_type& key) { return rep.erase(key); } - void erase(iterator it) { rep.erase(it); } - void erase(iterator f, iterator l) { rep.erase(f, l); } - - - // Comparison - bool operator==(const sparse_hash_set& hs) const { return rep == hs.rep; } - bool operator!=(const sparse_hash_set& hs) const { return rep != hs.rep; } - - - // I/O -- this is an add-on for writing metainformation to disk - // - // For maximum flexibility, this does not assume a particular - // file type (though it will probably be a FILE *). We just pass - // the fp through to rep. - - // If your keys and values are simple enough, you can pass this - // serializer to serialize()/unserialize(). "Simple enough" means - // value_type is a POD type that contains no pointers. Note, - // however, we don't try to normalize endianness. - typedef typename ht::NopointerSerializer NopointerSerializer; - - // serializer: a class providing operator()(OUTPUT*, const value_type&) - // (writing value_type to OUTPUT). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a - // pointer to a class providing size_t Write(const void*, size_t), - // which writes a buffer into a stream (which fp presumably - // owns) and returns the number of bytes successfully written. - // Note basic_ostream<not_char> is not currently supported. - template <typename ValueSerializer, typename OUTPUT> - bool serialize(ValueSerializer serializer, OUTPUT* fp) { - return rep.serialize(serializer, fp); - } - - // serializer: a functor providing operator()(INPUT*, value_type*) - // (reading from INPUT and into value_type). You can specify a - // NopointerSerializer object if appropriate (see above). - // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a - // pointer to a class providing size_t Read(void*, size_t), - // which reads into a buffer from a stream (which fp presumably - // owns) and returns the number of bytes successfully read. - // Note basic_istream<not_char> is not currently supported. - // NOTE: Since value_type is const Key, ValueSerializer - // may need to do a const cast in order to fill in the key. - // NOTE: if Key is not a POD type, the serializer MUST use - // placement-new to initialize its value, rather than a normal - // equals-assignment or similar. (The value_type* passed into - // the serializer points to garbage memory.) - template <typename ValueSerializer, typename INPUT> - bool unserialize(ValueSerializer serializer, INPUT* fp) { - return rep.unserialize(serializer, fp); - } - - // The four methods below are DEPRECATED. - // Use serialize() and unserialize() for new code. - template <typename OUTPUT> - bool write_metadata(OUTPUT *fp) { return rep.write_metadata(fp); } - - template <typename INPUT> - bool read_metadata(INPUT *fp) { return rep.read_metadata(fp); } - - template <typename OUTPUT> - bool write_nopointer_data(OUTPUT *fp) { return rep.write_nopointer_data(fp); } - - template <typename INPUT> - bool read_nopointer_data(INPUT *fp) { return rep.read_nopointer_data(fp); } -}; - -template <class Val, class HashFcn, class EqualKey, class Alloc> -inline void swap(sparse_hash_set<Val, HashFcn, EqualKey, Alloc>& hs1, - sparse_hash_set<Val, HashFcn, EqualKey, Alloc>& hs2) { - hs1.swap(hs2); -} - -_END_GOOGLE_NAMESPACE_ - -#endif /* _SPARSE_HASH_SET_H_ */ +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// +// This is just a very thin wrapper over sparsehashtable.h, just +// like sgi stl's stl_hash_set is a very thin wrapper over +// stl_hashtable. The major thing we define is operator[], because +// we have a concept of a data_type which stl_hashtable doesn't +// (it only has a key and a value). +// +// This is more different from sparse_hash_map than you might think, +// because all iterators for sets are const (you obviously can't +// change the key, and for sets there is no value). +// +// We adhere mostly to the STL semantics for hash-map. One important +// exception is that insert() may invalidate iterators entirely -- STL +// semantics are that insert() may reorder iterators, but they all +// still refer to something valid in the hashtable. Not so for us. +// Likewise, insert() may invalidate pointers into the hashtable. +// (Whether insert invalidates iterators and pointers depends on +// whether it results in a hashtable resize). On the plus side, +// delete() doesn't invalidate iterators or pointers at all, or even +// change the ordering of elements. +// +// Here are a few "power user" tips: +// +// 1) set_deleted_key(): +// Unlike STL's hash_map, if you want to use erase() you +// *must* call set_deleted_key() after construction. +// +// 2) resize(0): +// When an item is deleted, its memory isn't freed right +// away. This allows you to iterate over a hashtable, +// and call erase(), without invalidating the iterator. +// To force the memory to be freed, call resize(0). +// For tr1 compatibility, this can also be called as rehash(0). +// +// 3) min_load_factor(0.0) +// Setting the minimum load factor to 0.0 guarantees that +// the hash table will never shrink. +// +// Roughly speaking: +// (1) dense_hash_set: fastest, uses the most memory unless entries are small +// (2) sparse_hash_set: slowest, uses the least memory +// (3) hash_set / unordered_set (STL): in the middle +// +// Typically I use sparse_hash_set when I care about space and/or when +// I need to save the hashtable on disk. I use hash_set otherwise. I +// don't personally use dense_hash_set ever; some people use it for +// small sets with lots of lookups. +// +// - dense_hash_set has, typically, about 78% memory overhead (if your +// data takes up X bytes, the hash_set uses .78X more bytes in overhead). +// - sparse_hash_set has about 4 bits overhead per entry. +// - sparse_hash_set can be 3-7 times slower than the others for lookup and, +// especially, inserts. See time_hash_map.cc for details. +// +// See /usr/(local/)?doc/sparsehash-*/sparse_hash_set.html +// for information about how to use this class. + +#ifndef _SPARSE_HASH_SET_H_ +#define _SPARSE_HASH_SET_H_ + +#include <sparsehash/internal/sparseconfig.h> +#include <algorithm> // needed by stl_alloc +#include <functional> // for equal_to<> +#include <memory> // for alloc (which we don't use) +#include <utility> // for pair<> +#include <sparsehash/internal/libc_allocator_with_realloc.h> +#include <sparsehash/internal/sparsehashtable.h> // IWYU pragma: export +#include HASH_FUN_H // for hash<> + +_START_GOOGLE_NAMESPACE_ + +template <class Value, + class HashFcn = SPARSEHASH_HASH<Value>, // defined in sparseconfig.h + class EqualKey = std::equal_to<Value>, + class Alloc = libc_allocator_with_realloc<Value> > +class sparse_hash_set { + private: + // Apparently identity is not stl-standard, so we define our own + struct Identity { + typedef const Value& result_type; + const Value& operator()(const Value& v) const { return v; } + }; + struct SetKey { + void operator()(Value* value, const Value& new_key) const { + *value = new_key; + } + }; + + typedef sparse_hashtable<Value, Value, HashFcn, Identity, SetKey, + EqualKey, Alloc> ht; + ht rep; + + public: + typedef typename ht::key_type key_type; + typedef typename ht::value_type value_type; + typedef typename ht::hasher hasher; + typedef typename ht::key_equal key_equal; + typedef Alloc allocator_type; + + typedef typename ht::size_type size_type; + typedef typename ht::difference_type difference_type; + typedef typename ht::const_pointer pointer; + typedef typename ht::const_pointer const_pointer; + typedef typename ht::const_reference reference; + typedef typename ht::const_reference const_reference; + + typedef typename ht::const_iterator iterator; + typedef typename ht::const_iterator const_iterator; + typedef typename ht::const_local_iterator local_iterator; + typedef typename ht::const_local_iterator const_local_iterator; + + + // Iterator functions -- recall all iterators are const + iterator begin() const { return rep.begin(); } + iterator end() const { return rep.end(); } + + // These come from tr1's unordered_set. For us, a bucket has 0 or 1 elements. + local_iterator begin(size_type i) const { return rep.begin(i); } + local_iterator end(size_type i) const { return rep.end(i); } + + + // Accessor functions + allocator_type get_allocator() const { return rep.get_allocator(); } + hasher hash_funct() const { return rep.hash_funct(); } + hasher hash_function() const { return hash_funct(); } // tr1 name + key_equal key_eq() const { return rep.key_eq(); } + + + // Constructors + explicit sparse_hash_set(size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { + } + + template <class InputIterator> + sparse_hash_set(InputIterator f, InputIterator l, + size_type expected_max_items_in_table = 0, + const hasher& hf = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : rep(expected_max_items_in_table, hf, eql, Identity(), SetKey(), alloc) { + rep.insert(f, l); + } + // We use the default copy constructor + // We use the default operator=() + // We use the default destructor + + void clear() { rep.clear(); } + void swap(sparse_hash_set& hs) { rep.swap(hs.rep); } + + + // Functions concerning size + size_type size() const { return rep.size(); } + size_type max_size() const { return rep.max_size(); } + bool empty() const { return rep.empty(); } + size_type bucket_count() const { return rep.bucket_count(); } + size_type max_bucket_count() const { return rep.max_bucket_count(); } + + // These are tr1 methods. bucket() is the bucket the key is or would be in. + size_type bucket_size(size_type i) const { return rep.bucket_size(i); } + size_type bucket(const key_type& key) const { return rep.bucket(key); } + float load_factor() const { + return size() * 1.0f / bucket_count(); + } + float max_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return grow; + } + void max_load_factor(float new_grow) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(shrink, new_grow); + } + // These aren't tr1 methods but perhaps ought to be. + float min_load_factor() const { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + return shrink; + } + void min_load_factor(float new_shrink) { + float shrink, grow; + rep.get_resizing_parameters(&shrink, &grow); + rep.set_resizing_parameters(new_shrink, grow); + } + // Deprecated; use min_load_factor() or max_load_factor() instead. + void set_resizing_parameters(float shrink, float grow) { + rep.set_resizing_parameters(shrink, grow); + } + + void resize(size_type hint) { rep.resize(hint); } + void rehash(size_type hint) { resize(hint); } // the tr1 name + + // Lookup routines + iterator find(const key_type& key) const { return rep.find(key); } + + size_type count(const key_type& key) const { return rep.count(key); } + + std::pair<iterator, iterator> equal_range(const key_type& key) const { + return rep.equal_range(key); + } + + + // Insertion routines + std::pair<iterator, bool> insert(const value_type& obj) { + std::pair<typename ht::iterator, bool> p = rep.insert(obj); + return std::pair<iterator, bool>(p.first, p.second); // const to non-const + } + template <class InputIterator> void insert(InputIterator f, InputIterator l) { + rep.insert(f, l); + } + void insert(const_iterator f, const_iterator l) { + rep.insert(f, l); + } + // Required for std::insert_iterator; the passed-in iterator is ignored. + iterator insert(iterator, const value_type& obj) { + return insert(obj).first; + } + + // Deletion routines + // THESE ARE NON-STANDARD! I make you specify an "impossible" key + // value to identify deleted buckets. You can change the key as + // time goes on, or get rid of it entirely to be insert-only. + void set_deleted_key(const key_type& key) { rep.set_deleted_key(key); } + void clear_deleted_key() { rep.clear_deleted_key(); } + key_type deleted_key() const { return rep.deleted_key(); } + + // These are standard + size_type erase(const key_type& key) { return rep.erase(key); } + void erase(iterator it) { rep.erase(it); } + void erase(iterator f, iterator l) { rep.erase(f, l); } + + + // Comparison + bool operator==(const sparse_hash_set& hs) const { return rep == hs.rep; } + bool operator!=(const sparse_hash_set& hs) const { return rep != hs.rep; } + + + // I/O -- this is an add-on for writing metainformation to disk + // + // For maximum flexibility, this does not assume a particular + // file type (though it will probably be a FILE *). We just pass + // the fp through to rep. + + // If your keys and values are simple enough, you can pass this + // serializer to serialize()/unserialize(). "Simple enough" means + // value_type is a POD type that contains no pointers. Note, + // however, we don't try to normalize endianness. + typedef typename ht::NopointerSerializer NopointerSerializer; + + // serializer: a class providing operator()(OUTPUT*, const value_type&) + // (writing value_type to OUTPUT). You can specify a + // NopointerSerializer object if appropriate (see above). + // fp: either a FILE*, OR an ostream*/subclass_of_ostream*, OR a + // pointer to a class providing size_t Write(const void*, size_t), + // which writes a buffer into a stream (which fp presumably + // owns) and returns the number of bytes successfully written. + // Note basic_ostream<not_char> is not currently supported. + template <typename ValueSerializer, typename OUTPUT> + bool serialize(ValueSerializer serializer, OUTPUT* fp) { + return rep.serialize(serializer, fp); + } + + // serializer: a functor providing operator()(INPUT*, value_type*) + // (reading from INPUT and into value_type). You can specify a + // NopointerSerializer object if appropriate (see above). + // fp: either a FILE*, OR an istream*/subclass_of_istream*, OR a + // pointer to a class providing size_t Read(void*, size_t), + // which reads into a buffer from a stream (which fp presumably + // owns) and returns the number of bytes successfully read. + // Note basic_istream<not_char> is not currently supported. + // NOTE: Since value_type is const Key, ValueSerializer + // may need to do a const cast in order to fill in the key. + // NOTE: if Key is not a POD type, the serializer MUST use + // placement-new to initialize its value, rather than a normal + // equals-assignment or similar. (The value_type* passed into + // the serializer points to garbage memory.) + template <typename ValueSerializer, typename INPUT> + bool unserialize(ValueSerializer serializer, INPUT* fp) { + return rep.unserialize(serializer, fp); + } + + // The four methods below are DEPRECATED. + // Use serialize() and unserialize() for new code. + template <typename OUTPUT> + bool write_metadata(OUTPUT *fp) { return rep.write_metadata(fp); } + + template <typename INPUT> + bool read_metadata(INPUT *fp) { return rep.read_metadata(fp); } + + template <typename OUTPUT> + bool write_nopointer_data(OUTPUT *fp) { return rep.write_nopointer_data(fp); } + + template <typename INPUT> + bool read_nopointer_data(INPUT *fp) { return rep.read_nopointer_data(fp); } +}; + +template <class Val, class HashFcn, class EqualKey, class Alloc> +inline void swap(sparse_hash_set<Val, HashFcn, EqualKey, Alloc>& hs1, + sparse_hash_set<Val, HashFcn, EqualKey, Alloc>& hs2) { + hs1.swap(hs2); +} + +_END_GOOGLE_NAMESPACE_ + +#endif /* _SPARSE_HASH_SET_H_ */ diff --git a/contrib/libs/sparsehash/src/sparsehash/sparsetable b/contrib/libs/sparsehash/src/sparsehash/sparsetable index 11fd3990ae..6259ebdb04 100644 --- a/contrib/libs/sparsehash/src/sparsehash/sparsetable +++ b/contrib/libs/sparsehash/src/sparsehash/sparsetable @@ -1,1825 +1,1825 @@ -// Copyright (c) 2005, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// --- -// -// -// A sparsetable is a random container that implements a sparse array, -// that is, an array that uses very little memory to store unassigned -// indices (in this case, between 1-2 bits per unassigned index). For -// instance, if you allocate an array of size 5 and assign a[2] = <big -// struct>, then a[2] will take up a lot of memory but a[0], a[1], -// a[3], and a[4] will not. Array elements that have a value are -// called "assigned". Array elements that have no value yet, or have -// had their value cleared using erase() or clear(), are called -// "unassigned". -// -// Unassigned values seem to have the default value of T (see below). -// Nevertheless, there is a difference between an unassigned index and -// one explicitly assigned the value of T(). The latter is considered -// assigned. -// -// Access to an array element is constant time, as is insertion and -// deletion. Insertion and deletion may be fairly slow, however: -// because of this container's memory economy, each insert and delete -// causes a memory reallocation. -// -// NOTE: You should not test(), get(), or set() any index that is -// greater than sparsetable.size(). If you need to do that, call -// resize() first. -// -// --- Template parameters -// PARAMETER DESCRIPTION DEFAULT -// T The value of the array: the type of -- -// object that is stored in the array. -// -// GROUP_SIZE How large each "group" in the table 48 -// is (see below). Larger values use -// a little less memory but cause most -// operations to be a little slower -// -// Alloc: Allocator to use to allocate memory. libc_allocator_with_realloc -// -// --- Model of -// Random Access Container -// -// --- Type requirements -// T must be Copy Constructible. It need not be Assignable. -// -// --- Public base classes -// None. -// -// --- Members -// Type members -// -// MEMBER WHERE DEFINED DESCRIPTION -// value_type container The type of object, T, stored in the array -// allocator_type container Allocator to use -// pointer container Pointer to p -// const_pointer container Const pointer to p -// reference container Reference to t -// const_reference container Const reference to t -// size_type container An unsigned integral type -// difference_type container A signed integral type -// iterator [*] container Iterator used to iterate over a sparsetable -// const_iterator container Const iterator used to iterate over a table -// reverse_iterator reversible Iterator used to iterate backwards over -// container a sparsetable -// const_reverse_iterator reversible container Guess -// nonempty_iterator [+] sparsetable Iterates over assigned -// array elements only -// const_nonempty_iterator sparsetable Iterates over assigned -// array elements only -// reverse_nonempty_iterator sparsetable Iterates backwards over -// assigned array elements only -// const_reverse_nonempty_iterator sparsetable Iterates backwards over -// assigned array elements only -// -// [*] All iterators are const in a sparsetable (though nonempty_iterators -// may not be). Use get() and set() to assign values, not iterators. -// -// [+] iterators are random-access iterators. nonempty_iterators are -// bidirectional iterators. - -// Iterator members -// MEMBER WHERE DEFINED DESCRIPTION -// -// iterator begin() container An iterator to the beginning of the table -// iterator end() container An iterator to the end of the table -// const_iterator container A const_iterator pointing to the -// begin() const beginning of a sparsetable -// const_iterator container A const_iterator pointing to the -// end() const end of a sparsetable -// -// reverse_iterator reversable Points to beginning of a reversed -// rbegin() container sparsetable -// reverse_iterator reversable Points to end of a reversed table -// rend() container -// const_reverse_iterator reversable Points to beginning of a -// rbegin() const container reversed sparsetable -// const_reverse_iterator reversable Points to end of a reversed table -// rend() const container -// -// nonempty_iterator sparsetable Points to first assigned element -// begin() of a sparsetable -// nonempty_iterator sparsetable Points past last assigned element -// end() of a sparsetable -// const_nonempty_iterator sparsetable Points to first assigned element -// begin() const of a sparsetable -// const_nonempty_iterator sparsetable Points past last assigned element -// end() const of a sparsetable -// -// reverse_nonempty_iterator sparsetable Points to first assigned element -// begin() of a reversed sparsetable -// reverse_nonempty_iterator sparsetable Points past last assigned element -// end() of a reversed sparsetable -// const_reverse_nonempty_iterator sparsetable Points to first assigned -// begin() const elt of a reversed sparsetable -// const_reverse_nonempty_iterator sparsetable Points past last assigned -// end() const elt of a reversed sparsetable -// -// -// Other members -// MEMBER WHERE DEFINED DESCRIPTION -// sparsetable() sparsetable A table of size 0; must resize() -// before using. -// sparsetable(size_type size) sparsetable A table of size size. All -// indices are unassigned. -// sparsetable( -// const sparsetable &tbl) sparsetable Copy constructor -// ~sparsetable() sparsetable The destructor -// sparsetable &operator=( sparsetable The assignment operator -// const sparsetable &tbl) -// -// void resize(size_type size) sparsetable Grow or shrink a table to -// have size indices [*] -// -// void swap(sparsetable &x) sparsetable Swap two sparsetables -// void swap(sparsetable &x, sparsetable Swap two sparsetables -// sparsetable &y) (global, not member, function) -// -// size_type size() const sparsetable Number of "buckets" in the table -// size_type max_size() const sparsetable Max allowed size of a sparsetable -// bool empty() const sparsetable true if size() == 0 -// size_type num_nonempty() const sparsetable Number of assigned "buckets" -// -// const_reference get( sparsetable Value at index i, or default -// size_type i) const value if i is unassigned -// const_reference operator[]( sparsetable Identical to get(i) [+] -// difference_type i) const -// reference set(size_type i, sparsetable Set element at index i to -// const_reference val) be a copy of val -// bool test(size_type i) sparsetable True if element at index i -// const has been assigned to -// bool test(iterator pos) sparsetable True if element pointed to -// const by pos has been assigned to -// void erase(iterator pos) sparsetable Set element pointed to by -// pos to be unassigned [!] -// void erase(size_type i) sparsetable Set element i to be unassigned -// void erase(iterator start, sparsetable Erases all elements between -// iterator end) start and end -// void clear() sparsetable Erases all elements in the table -// -// I/O versions exist for both FILE* and for File* (Google2-style files): -// bool write_metadata(FILE *fp) sparsetable Writes a sparsetable to the -// bool write_metadata(File *fp) given file. true if write -// completes successfully -// bool read_metadata(FILE *fp) sparsetable Replaces sparsetable with -// bool read_metadata(File *fp) version read from fp. true -// if read completes sucessfully -// bool write_nopointer_data(FILE *fp) Read/write the data stored in -// bool read_nopointer_data(FILE*fp) the table, if it's simple -// -// bool operator==( forward Tests two tables for equality. -// const sparsetable &t1, container This is a global function, -// const sparsetable &t2) not a member function. -// bool operator<( forward Lexicographical comparison. -// const sparsetable &t1, container This is a global function, -// const sparsetable &t2) not a member function. -// -// [*] If you shrink a sparsetable using resize(), assigned elements -// past the end of the table are removed using erase(). If you grow -// a sparsetable, new unassigned indices are created. -// -// [+] Note that operator[] returns a const reference. You must use -// set() to change the value of a table element. -// -// [!] Unassignment also calls the destructor. -// -// Iterators are invalidated whenever an item is inserted or -// deleted (ie set() or erase() is used) or when the size of -// the table changes (ie resize() or clear() is used). -// -// See doc/sparsetable.html for more information about how to use this class. - -// Note: this uses STL style for naming, rather than Google naming. -// That's because this is an STL-y container - -#ifndef UTIL_GTL_SPARSETABLE_H_ -#define UTIL_GTL_SPARSETABLE_H_ - -#include <sparsehash/internal/sparseconfig.h> -#include <stdlib.h> // for malloc/free -#include <stdio.h> // to read/write tables -#include <string.h> // for memcpy -#ifdef HAVE_STDINT_H -#include <stdint.h> // the normal place uint16_t is defined -#endif -#ifdef HAVE_SYS_TYPES_H -#include <sys/types.h> // the normal place u_int16_t is defined -#endif -#ifdef HAVE_INTTYPES_H -#include <inttypes.h> // a third place for uint16_t or u_int16_t -#endif -#include <assert.h> // for bounds checking -#include <iterator> // to define reverse_iterator for me -#include <algorithm> // equal, lexicographical_compare, swap,... -#include <memory> // uninitialized_copy, uninitialized_fill -#include <vector> // a sparsetable is a vector of groups -#include <sparsehash/type_traits.h> -#include <sparsehash/internal/hashtable-common.h> -#include <sparsehash/internal/libc_allocator_with_realloc.h> - -// A lot of work to get a type that's guaranteed to be 16 bits... -#ifndef HAVE_U_INT16_T -# if defined HAVE_UINT16_T - typedef uint16_t u_int16_t; // true on solaris, possibly other C99 libc's -# elif defined HAVE___UINT16 - typedef __int16 int16_t; // true on vc++7 - typedef unsigned __int16 u_int16_t; -# else - // Cannot find a 16-bit integer type. Hoping for the best with "short"... - typedef short int int16_t; - typedef unsigned short int u_int16_t; -# endif -#endif - -_START_GOOGLE_NAMESPACE_ - -namespace base { // just to make google->opensource transition easier -using GOOGLE_NAMESPACE::true_type; -using GOOGLE_NAMESPACE::false_type; -using GOOGLE_NAMESPACE::integral_constant; -using GOOGLE_NAMESPACE::has_trivial_copy; -using GOOGLE_NAMESPACE::has_trivial_destructor; -using GOOGLE_NAMESPACE::is_same; -} - - -// The smaller this is, the faster lookup is (because the group bitmap is -// smaller) and the faster insert is, because there's less to move. -// On the other hand, there are more groups. Since group::size_type is -// a short, this number should be of the form 32*x + 16 to avoid waste. -static const u_int16_t DEFAULT_SPARSEGROUP_SIZE = 48; // fits in 1.5 words - - -// Our iterator as simple as iterators can be: basically it's just -// the index into our table. Dereference, the only complicated -// thing, we punt to the table class. This just goes to show how -// much machinery STL requires to do even the most trivial tasks. -// -// A NOTE ON ASSIGNING: -// A sparse table does not actually allocate memory for entries -// that are not filled. Because of this, it becomes complicated -// to have a non-const iterator: we don't know, if the iterator points -// to a not-filled bucket, whether you plan to fill it with something -// or whether you plan to read its value (in which case you'll get -// the default bucket value). Therefore, while we can define const -// operations in a pretty 'normal' way, for non-const operations, we -// define something that returns a helper object with operator= and -// operator& that allocate a bucket lazily. We use this for table[] -// and also for regular table iterators. - -template <class tabletype> -class table_element_adaptor { - public: - typedef typename tabletype::value_type value_type; - typedef typename tabletype::size_type size_type; - typedef typename tabletype::reference reference; - typedef typename tabletype::pointer pointer; - - table_element_adaptor(tabletype *tbl, size_type p) - : table(tbl), pos(p) { } - table_element_adaptor& operator= (const value_type &val) { - table->set(pos, val); - return *this; - } - operator value_type() { return table->get(pos); } // we look like a value - pointer operator& () { return &table->mutating_get(pos); } - - private: - tabletype* table; - size_type pos; -}; - -// Our iterator as simple as iterators can be: basically it's just -// the index into our table. Dereference, the only complicated -// thing, we punt to the table class. This just goes to show how -// much machinery STL requires to do even the most trivial tasks. -// -// By templatizing over tabletype, we have one iterator type which -// we can use for both sparsetables and sparsebins. In fact it -// works on any class that allows size() and operator[] (eg vector), -// as long as it does the standard STL typedefs too (eg value_type). - -template <class tabletype> -class table_iterator { - public: - typedef table_iterator iterator; - - typedef std::random_access_iterator_tag iterator_category; - typedef typename tabletype::value_type value_type; - typedef typename tabletype::difference_type difference_type; - typedef typename tabletype::size_type size_type; - typedef table_element_adaptor<tabletype> reference; - typedef table_element_adaptor<tabletype>* pointer; - - // The "real" constructor - table_iterator(tabletype *tbl, size_type p) - : table(tbl), pos(p) { } - // The default constructor, used when I define vars of type table::iterator - table_iterator() : table(NULL), pos(0) { } - // The copy constructor, for when I say table::iterator foo = tbl.begin() - // The default destructor is fine; we don't define one - // The default operator= is fine; we don't define one - - // The main thing our iterator does is dereference. If the table entry - // we point to is empty, we return the default value type. - // This is the big different function from the const iterator. - reference operator*() { - return table_element_adaptor<tabletype>(table, pos); - } - pointer operator->() { return &(operator*()); } - - // Helper function to assert things are ok; eg pos is still in range - void check() const { - assert(table); - assert(pos <= table->size()); - } - - // Arithmetic: we just do arithmetic on pos. We don't even need to - // do bounds checking, since STL doesn't consider that its job. :-) - iterator& operator+=(size_type t) { pos += t; check(); return *this; } - iterator& operator-=(size_type t) { pos -= t; check(); return *this; } - iterator& operator++() { ++pos; check(); return *this; } - iterator& operator--() { --pos; check(); return *this; } - iterator operator++(int) { iterator tmp(*this); // for x++ - ++pos; check(); return tmp; } - iterator operator--(int) { iterator tmp(*this); // for x-- - --pos; check(); return tmp; } - iterator operator+(difference_type i) const { iterator tmp(*this); - tmp += i; return tmp; } - iterator operator-(difference_type i) const { iterator tmp(*this); - tmp -= i; return tmp; } - difference_type operator-(iterator it) const { // for "x = it2 - it" - assert(table == it.table); - return pos - it.pos; - } - reference operator[](difference_type n) const { - return *(*this + n); // simple though not totally efficient - } - - // Comparisons. - bool operator==(const iterator& it) const { - return table == it.table && pos == it.pos; - } - bool operator<(const iterator& it) const { - assert(table == it.table); // life is bad bad bad otherwise - return pos < it.pos; - } - bool operator!=(const iterator& it) const { return !(*this == it); } - bool operator<=(const iterator& it) const { return !(it < *this); } - bool operator>(const iterator& it) const { return it < *this; } - bool operator>=(const iterator& it) const { return !(*this < it); } - - // Here's the info we actually need to be an iterator - tabletype *table; // so we can dereference and bounds-check - size_type pos; // index into the table -}; - -// support for "3 + iterator" has to be defined outside the class, alas -template<class T> -table_iterator<T> operator+(typename table_iterator<T>::difference_type i, - table_iterator<T> it) { - return it + i; // so people can say it2 = 3 + it -} - -template <class tabletype> -class const_table_iterator { - public: - typedef table_iterator<tabletype> iterator; - typedef const_table_iterator const_iterator; - - typedef std::random_access_iterator_tag iterator_category; - typedef typename tabletype::value_type value_type; - typedef typename tabletype::difference_type difference_type; - typedef typename tabletype::size_type size_type; - typedef typename tabletype::const_reference reference; // we're const-only - typedef typename tabletype::const_pointer pointer; - - // The "real" constructor - const_table_iterator(const tabletype *tbl, size_type p) - : table(tbl), pos(p) { } - // The default constructor, used when I define vars of type table::iterator - const_table_iterator() : table(NULL), pos(0) { } - // The copy constructor, for when I say table::iterator foo = tbl.begin() - // Also converts normal iterators to const iterators - const_table_iterator(const iterator &from) - : table(from.table), pos(from.pos) { } - // The default destructor is fine; we don't define one - // The default operator= is fine; we don't define one - - // The main thing our iterator does is dereference. If the table entry - // we point to is empty, we return the default value type. - reference operator*() const { return (*table)[pos]; } - pointer operator->() const { return &(operator*()); } - - // Helper function to assert things are ok; eg pos is still in range - void check() const { - assert(table); - assert(pos <= table->size()); - } - - // Arithmetic: we just do arithmetic on pos. We don't even need to - // do bounds checking, since STL doesn't consider that its job. :-) - const_iterator& operator+=(size_type t) { pos += t; check(); return *this; } - const_iterator& operator-=(size_type t) { pos -= t; check(); return *this; } - const_iterator& operator++() { ++pos; check(); return *this; } - const_iterator& operator--() { --pos; check(); return *this; } - const_iterator operator++(int) { const_iterator tmp(*this); // for x++ - ++pos; check(); return tmp; } - const_iterator operator--(int) { const_iterator tmp(*this); // for x-- - --pos; check(); return tmp; } - const_iterator operator+(difference_type i) const { const_iterator tmp(*this); - tmp += i; return tmp; } - const_iterator operator-(difference_type i) const { const_iterator tmp(*this); - tmp -= i; return tmp; } - difference_type operator-(const_iterator it) const { // for "x = it2 - it" - assert(table == it.table); - return pos - it.pos; - } - reference operator[](difference_type n) const { - return *(*this + n); // simple though not totally efficient - } - - // Comparisons. - bool operator==(const const_iterator& it) const { - return table == it.table && pos == it.pos; - } - bool operator<(const const_iterator& it) const { - assert(table == it.table); // life is bad bad bad otherwise - return pos < it.pos; - } - bool operator!=(const const_iterator& it) const { return !(*this == it); } - bool operator<=(const const_iterator& it) const { return !(it < *this); } - bool operator>(const const_iterator& it) const { return it < *this; } - bool operator>=(const const_iterator& it) const { return !(*this < it); } - - // Here's the info we actually need to be an iterator - const tabletype *table; // so we can dereference and bounds-check - size_type pos; // index into the table -}; - -// support for "3 + iterator" has to be defined outside the class, alas -template<class T> -const_table_iterator<T> operator+(typename - const_table_iterator<T>::difference_type i, - const_table_iterator<T> it) { - return it + i; // so people can say it2 = 3 + it -} - - -// --------------------------------------------------------------------------- - - -/* -// This is a 2-D iterator. You specify a begin and end over a list -// of *containers*. We iterate over each container by iterating over -// it. It's actually simple: -// VECTOR.begin() VECTOR[0].begin() --------> VECTOR[0].end() ---, -// | ________________________________________________/ -// | \_> VECTOR[1].begin() --------> VECTOR[1].end() -, -// | ___________________________________________________/ -// v \_> ...... -// VECTOR.end() -// -// It's impossible to do random access on one of these things in constant -// time, so it's just a bidirectional iterator. -// -// Unfortunately, because we need to use this for a non-empty iterator, -// we use nonempty_begin() and nonempty_end() instead of begin() and end() -// (though only going across, not down). -*/ - -#define TWOD_BEGIN_ nonempty_begin -#define TWOD_END_ nonempty_end -#define TWOD_ITER_ nonempty_iterator -#define TWOD_CONST_ITER_ const_nonempty_iterator - -template <class containertype> -class two_d_iterator { - public: - typedef two_d_iterator iterator; - - typedef std::bidirectional_iterator_tag iterator_category; - // apparently some versions of VC++ have trouble with two ::'s in a typename - typedef typename containertype::value_type _tmp_vt; - typedef typename _tmp_vt::value_type value_type; - typedef typename _tmp_vt::difference_type difference_type; - typedef typename _tmp_vt::reference reference; - typedef typename _tmp_vt::pointer pointer; - - // The "real" constructor. begin and end specify how many rows we have - // (in the diagram above); we always iterate over each row completely. - two_d_iterator(typename containertype::iterator begin, - typename containertype::iterator end, - typename containertype::iterator curr) - : row_begin(begin), row_end(end), row_current(curr), col_current() { - if ( row_current != row_end ) { - col_current = row_current->TWOD_BEGIN_(); - advance_past_end(); // in case cur->begin() == cur->end() - } - } - // If you want to start at an arbitrary place, you can, I guess - two_d_iterator(typename containertype::iterator begin, - typename containertype::iterator end, - typename containertype::iterator curr, - typename containertype::value_type::TWOD_ITER_ col) - : row_begin(begin), row_end(end), row_current(curr), col_current(col) { - advance_past_end(); // in case cur->begin() == cur->end() - } - // The default constructor, used when I define vars of type table::iterator - two_d_iterator() : row_begin(), row_end(), row_current(), col_current() { } - // The default destructor is fine; we don't define one - // The default operator= is fine; we don't define one - - // Happy dereferencer - reference operator*() const { return *col_current; } - pointer operator->() const { return &(operator*()); } - - // Arithmetic: we just do arithmetic on pos. We don't even need to - // do bounds checking, since STL doesn't consider that its job. :-) - // NOTE: this is not amortized constant time! What do we do about it? - void advance_past_end() { // used when col_current points to end() - while ( col_current == row_current->TWOD_END_() ) { // end of current row - ++row_current; // go to beginning of next - if ( row_current != row_end ) // col is irrelevant at end - col_current = row_current->TWOD_BEGIN_(); - else - break; // don't go past row_end - } - } - - iterator& operator++() { - assert(row_current != row_end); // how to ++ from there? - ++col_current; - advance_past_end(); // in case col_current is at end() - return *this; - } - iterator& operator--() { - while ( row_current == row_end || - col_current == row_current->TWOD_BEGIN_() ) { - assert(row_current != row_begin); - --row_current; - col_current = row_current->TWOD_END_(); // this is 1 too far - } - --col_current; - return *this; - } - iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } - iterator operator--(int) { iterator tmp(*this); --*this; return tmp; } - - - // Comparisons. - bool operator==(const iterator& it) const { - return ( row_begin == it.row_begin && - row_end == it.row_end && - row_current == it.row_current && - (row_current == row_end || col_current == it.col_current) ); - } - bool operator!=(const iterator& it) const { return !(*this == it); } - - - // Here's the info we actually need to be an iterator - // These need to be public so we convert from iterator to const_iterator - typename containertype::iterator row_begin, row_end, row_current; - typename containertype::value_type::TWOD_ITER_ col_current; -}; - -// The same thing again, but this time const. :-( -template <class containertype> -class const_two_d_iterator { - public: - typedef const_two_d_iterator iterator; - - typedef std::bidirectional_iterator_tag iterator_category; - // apparently some versions of VC++ have trouble with two ::'s in a typename - typedef typename containertype::value_type _tmp_vt; - typedef typename _tmp_vt::value_type value_type; - typedef typename _tmp_vt::difference_type difference_type; - typedef typename _tmp_vt::const_reference reference; - typedef typename _tmp_vt::const_pointer pointer; - - const_two_d_iterator(typename containertype::const_iterator begin, - typename containertype::const_iterator end, - typename containertype::const_iterator curr) - : row_begin(begin), row_end(end), row_current(curr), col_current() { - if ( curr != end ) { - col_current = curr->TWOD_BEGIN_(); - advance_past_end(); // in case cur->begin() == cur->end() - } - } - const_two_d_iterator(typename containertype::const_iterator begin, - typename containertype::const_iterator end, - typename containertype::const_iterator curr, - typename containertype::value_type::TWOD_CONST_ITER_ col) - : row_begin(begin), row_end(end), row_current(curr), col_current(col) { - advance_past_end(); // in case cur->begin() == cur->end() - } - const_two_d_iterator() - : row_begin(), row_end(), row_current(), col_current() { - } - // Need this explicitly so we can convert normal iterators to const iterators - const_two_d_iterator(const two_d_iterator<containertype>& it) : - row_begin(it.row_begin), row_end(it.row_end), row_current(it.row_current), - col_current(it.col_current) { } - - typename containertype::const_iterator row_begin, row_end, row_current; - typename containertype::value_type::TWOD_CONST_ITER_ col_current; - - - // EVERYTHING FROM HERE DOWN IS THE SAME AS THE NON-CONST ITERATOR - reference operator*() const { return *col_current; } - pointer operator->() const { return &(operator*()); } - - void advance_past_end() { // used when col_current points to end() - while ( col_current == row_current->TWOD_END_() ) { // end of current row - ++row_current; // go to beginning of next - if ( row_current != row_end ) // col is irrelevant at end - col_current = row_current->TWOD_BEGIN_(); - else - break; // don't go past row_end - } - } - iterator& operator++() { - assert(row_current != row_end); // how to ++ from there? - ++col_current; - advance_past_end(); // in case col_current is at end() - return *this; - } - iterator& operator--() { - while ( row_current == row_end || - col_current == row_current->TWOD_BEGIN_() ) { - assert(row_current != row_begin); - --row_current; - col_current = row_current->TWOD_END_(); // this is 1 too far - } - --col_current; - return *this; - } - iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } - iterator operator--(int) { iterator tmp(*this); --*this; return tmp; } - - bool operator==(const iterator& it) const { - return ( row_begin == it.row_begin && - row_end == it.row_end && - row_current == it.row_current && - (row_current == row_end || col_current == it.col_current) ); - } - bool operator!=(const iterator& it) const { return !(*this == it); } -}; - -// We provide yet another version, to be as frugal with memory as -// possible. This one frees each block of memory as it finishes -// iterating over it. By the end, the entire table is freed. -// For understandable reasons, you can only iterate over it once, -// which is why it's an input iterator -template <class containertype> -class destructive_two_d_iterator { - public: - typedef destructive_two_d_iterator iterator; - - typedef std::input_iterator_tag iterator_category; - // apparently some versions of VC++ have trouble with two ::'s in a typename - typedef typename containertype::value_type _tmp_vt; - typedef typename _tmp_vt::value_type value_type; - typedef typename _tmp_vt::difference_type difference_type; - typedef typename _tmp_vt::reference reference; - typedef typename _tmp_vt::pointer pointer; - - destructive_two_d_iterator(typename containertype::iterator begin, - typename containertype::iterator end, - typename containertype::iterator curr) - : row_begin(begin), row_end(end), row_current(curr), col_current() { - if ( curr != end ) { - col_current = curr->TWOD_BEGIN_(); - advance_past_end(); // in case cur->begin() == cur->end() - } - } - destructive_two_d_iterator(typename containertype::iterator begin, - typename containertype::iterator end, - typename containertype::iterator curr, - typename containertype::value_type::TWOD_ITER_ col) - : row_begin(begin), row_end(end), row_current(curr), col_current(col) { - advance_past_end(); // in case cur->begin() == cur->end() - } - destructive_two_d_iterator() - : row_begin(), row_end(), row_current(), col_current() { - } - - typename containertype::iterator row_begin, row_end, row_current; - typename containertype::value_type::TWOD_ITER_ col_current; - - // This is the part that destroys - void advance_past_end() { // used when col_current points to end() - while ( col_current == row_current->TWOD_END_() ) { // end of current row - row_current->clear(); // the destructive part - // It would be nice if we could decrement sparsetable->num_buckets here - ++row_current; // go to beginning of next - if ( row_current != row_end ) // col is irrelevant at end - col_current = row_current->TWOD_BEGIN_(); - else - break; // don't go past row_end - } - } - - // EVERYTHING FROM HERE DOWN IS THE SAME AS THE REGULAR ITERATOR - reference operator*() const { return *col_current; } - pointer operator->() const { return &(operator*()); } - - iterator& operator++() { - assert(row_current != row_end); // how to ++ from there? - ++col_current; - advance_past_end(); // in case col_current is at end() - return *this; - } - iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } - - bool operator==(const iterator& it) const { - return ( row_begin == it.row_begin && - row_end == it.row_end && - row_current == it.row_current && - (row_current == row_end || col_current == it.col_current) ); - } - bool operator!=(const iterator& it) const { return !(*this == it); } -}; - -#undef TWOD_BEGIN_ -#undef TWOD_END_ -#undef TWOD_ITER_ -#undef TWOD_CONST_ITER_ - - - - -// SPARSE-TABLE -// ------------ -// The idea is that a table with (logically) t buckets is divided -// into t/M *groups* of M buckets each. (M is a constant set in -// GROUP_SIZE for efficiency.) Each group is stored sparsely. -// Thus, inserting into the table causes some array to grow, which is -// slow but still constant time. Lookup involves doing a -// logical-position-to-sparse-position lookup, which is also slow but -// constant time. The larger M is, the slower these operations are -// but the less overhead (slightly). -// -// To store the sparse array, we store a bitmap B, where B[i] = 1 iff -// bucket i is non-empty. Then to look up bucket i we really look up -// array[# of 1s before i in B]. This is constant time for fixed M. -// -// Terminology: the position of an item in the overall table (from -// 1 .. t) is called its "location." The logical position in a group -// (from 1 .. M ) is called its "position." The actual location in -// the array (from 1 .. # of non-empty buckets in the group) is -// called its "offset." - -template <class T, u_int16_t GROUP_SIZE, class Alloc> -class sparsegroup { - private: - typedef typename Alloc::template rebind<T>::other value_alloc_type; - - public: - // Basic types - typedef T value_type; - typedef Alloc allocator_type; - typedef typename value_alloc_type::reference reference; - typedef typename value_alloc_type::const_reference const_reference; - typedef typename value_alloc_type::pointer pointer; - typedef typename value_alloc_type::const_pointer const_pointer; - - typedef table_iterator<sparsegroup<T, GROUP_SIZE, Alloc> > iterator; - typedef const_table_iterator<sparsegroup<T, GROUP_SIZE, Alloc> > - const_iterator; - typedef table_element_adaptor<sparsegroup<T, GROUP_SIZE, Alloc> > - element_adaptor; - typedef u_int16_t size_type; // max # of buckets - typedef int16_t difference_type; - typedef std::reverse_iterator<const_iterator> const_reverse_iterator; - typedef std::reverse_iterator<iterator> reverse_iterator; // from iterator.h - - // These are our special iterators, that go over non-empty buckets in a - // group. These aren't const-only because you can change non-empty bcks. - typedef pointer nonempty_iterator; - typedef const_pointer const_nonempty_iterator; - typedef std::reverse_iterator<nonempty_iterator> reverse_nonempty_iterator; - typedef std::reverse_iterator<const_nonempty_iterator> const_reverse_nonempty_iterator; - - // Iterator functions - iterator begin() { return iterator(this, 0); } - const_iterator begin() const { return const_iterator(this, 0); } - iterator end() { return iterator(this, size()); } - const_iterator end() const { return const_iterator(this, size()); } - reverse_iterator rbegin() { return reverse_iterator(end()); } - const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } - reverse_iterator rend() { return reverse_iterator(begin()); } - const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } - - // We'll have versions for our special non-empty iterator too - nonempty_iterator nonempty_begin() { return group; } - const_nonempty_iterator nonempty_begin() const { return group; } - nonempty_iterator nonempty_end() { - return group + settings.num_buckets; - } - const_nonempty_iterator nonempty_end() const { - return group + settings.num_buckets; - } - reverse_nonempty_iterator nonempty_rbegin() { - return reverse_nonempty_iterator(nonempty_end()); - } - const_reverse_nonempty_iterator nonempty_rbegin() const { - return const_reverse_nonempty_iterator(nonempty_end()); - } - reverse_nonempty_iterator nonempty_rend() { - return reverse_nonempty_iterator(nonempty_begin()); - } - const_reverse_nonempty_iterator nonempty_rend() const { - return const_reverse_nonempty_iterator(nonempty_begin()); - } - - - // This gives us the "default" value to return for an empty bucket. - // We just use the default constructor on T, the template type - const_reference default_value() const { - static value_type defaultval = value_type(); - return defaultval; - } - - - private: - // We need to do all this bit manipulation, of course. ick - static size_type charbit(size_type i) { return i >> 3; } - static size_type modbit(size_type i) { return 1 << (i&7); } - int bmtest(size_type i) const { return bitmap[charbit(i)] & modbit(i); } - void bmset(size_type i) { bitmap[charbit(i)] |= modbit(i); } - void bmclear(size_type i) { bitmap[charbit(i)] &= ~modbit(i); } - - pointer allocate_group(size_type n) { - pointer retval = settings.allocate(n); - if (retval == NULL) { - // We really should use PRIuS here, but I don't want to have to add - // a whole new configure option, with concomitant macro namespace - // pollution, just to print this (unlikely) error message. So I cast. - fprintf(stderr, "sparsehash FATAL ERROR: failed to allocate %lu groups\n", - static_cast<unsigned long>(n)); - exit(1); - } - return retval; - } - - void free_group() { - if (!group) return; - pointer end_it = group + settings.num_buckets; - for (pointer p = group; p != end_it; ++p) - p->~value_type(); - settings.deallocate(group, settings.num_buckets); - group = NULL; - } - - static size_type bits_in_char(unsigned char c) { - // We could make these ints. The tradeoff is size (eg does it overwhelm - // the cache?) vs efficiency in referencing sub-word-sized array elements. - static const char bits_in[256] = { - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, - 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8, - }; - return bits_in[c]; - } - - public: // get_iter() in sparsetable needs it - // We need a small function that tells us how many set bits there are - // in positions 0..i-1 of the bitmap. It uses a big table. - // We make it static so templates don't allocate lots of these tables. - // There are lots of ways to do this calculation (called 'popcount'). - // The 8-bit table lookup is one of the fastest, though this - // implementation suffers from not doing any loop unrolling. See, eg, - // http://www.dalkescientific.com/writings/diary/archive/2008/07/03/hakmem_and_other_popcounts.html - // http://gurmeetsingh.wordpress.com/2008/08/05/fast-bit-counting-routines/ - static size_type pos_to_offset(const unsigned char *bm, size_type pos) { - size_type retval = 0; - - // [Note: condition pos > 8 is an optimization; convince yourself we - // give exactly the same result as if we had pos >= 8 here instead.] - for ( ; pos > 8; pos -= 8 ) // bm[0..pos/8-1] - retval += bits_in_char(*bm++); // chars we want *all* bits in - return retval + bits_in_char(*bm & ((1 << pos)-1)); // char including pos - } - - size_type pos_to_offset(size_type pos) const { // not static but still const - return pos_to_offset(bitmap, pos); - } - - // Returns the (logical) position in the bm[] array, i, such that - // bm[i] is the offset-th set bit in the array. It is the inverse - // of pos_to_offset. get_pos() uses this function to find the index - // of an nonempty_iterator in the table. Bit-twiddling from - // http://hackersdelight.org/basics.pdf - static size_type offset_to_pos(const unsigned char *bm, size_type offset) { - size_type retval = 0; - // This is sizeof(this->bitmap). - const size_type group_size = (GROUP_SIZE-1) / 8 + 1; - for (size_type i = 0; i < group_size; i++) { // forward scan - const size_type pop_count = bits_in_char(*bm); - if (pop_count > offset) { - unsigned char last_bm = *bm; - for (; offset > 0; offset--) { - last_bm &= (last_bm-1); // remove right-most set bit - } - // Clear all bits to the left of the rightmost bit (the &), - // and then clear the rightmost bit but set all bits to the - // right of it (the -1). - last_bm = (last_bm & -last_bm) - 1; - retval += bits_in_char(last_bm); - return retval; - } - offset -= pop_count; - retval += 8; - bm++; - } - return retval; - } - - size_type offset_to_pos(size_type offset) const { - return offset_to_pos(bitmap, offset); - } - - - public: - // Constructors -- default and copy -- and destructor - explicit sparsegroup(allocator_type& a) : - group(0), settings(alloc_impl<value_alloc_type>(a)) { - memset(bitmap, 0, sizeof(bitmap)); - } - sparsegroup(const sparsegroup& x) : group(0), settings(x.settings) { - if ( settings.num_buckets ) { - group = allocate_group(x.settings.num_buckets); - std::uninitialized_copy(x.group, x.group + x.settings.num_buckets, group); - } - memcpy(bitmap, x.bitmap, sizeof(bitmap)); - } - ~sparsegroup() { free_group(); } - - // Operator= is just like the copy constructor, I guess - // TODO(austern): Make this exception safe. Handle exceptions in value_type's - // copy constructor. - sparsegroup &operator=(const sparsegroup& x) { - if ( &x == this ) return *this; // x = x - if ( x.settings.num_buckets == 0 ) { - free_group(); - } else { - pointer p = allocate_group(x.settings.num_buckets); - std::uninitialized_copy(x.group, x.group + x.settings.num_buckets, p); - free_group(); - group = p; - } - memcpy(bitmap, x.bitmap, sizeof(bitmap)); - settings.num_buckets = x.settings.num_buckets; - return *this; - } - - // Many STL algorithms use swap instead of copy constructors - void swap(sparsegroup& x) { - std::swap(group, x.group); // defined in <algorithm> - for ( int i = 0; i < sizeof(bitmap) / sizeof(*bitmap); ++i ) - std::swap(bitmap[i], x.bitmap[i]); // swap not defined on arrays - std::swap(settings.num_buckets, x.settings.num_buckets); - // we purposefully don't swap the allocator, which may not be swap-able - } - - // It's always nice to be able to clear a table without deallocating it - void clear() { - free_group(); - memset(bitmap, 0, sizeof(bitmap)); - settings.num_buckets = 0; - } - - // Functions that tell you about size. Alas, these aren't so useful - // because our table is always fixed size. - size_type size() const { return GROUP_SIZE; } - size_type max_size() const { return GROUP_SIZE; } - bool empty() const { return false; } - // We also may want to know how many *used* buckets there are - size_type num_nonempty() const { return settings.num_buckets; } - - - // get()/set() are explicitly const/non-const. You can use [] if - // you want something that can be either (potentially more expensive). - const_reference get(size_type i) const { - if ( bmtest(i) ) // bucket i is occupied - return group[pos_to_offset(bitmap, i)]; - else - return default_value(); // return the default reference - } - - // TODO(csilvers): make protected + friend - // This is used by sparse_hashtable to get an element from the table - // when we know it exists. - const_reference unsafe_get(size_type i) const { - assert(bmtest(i)); - return group[pos_to_offset(bitmap, i)]; - } - - // TODO(csilvers): make protected + friend - reference mutating_get(size_type i) { // fills bucket i before getting - if ( !bmtest(i) ) - set(i, default_value()); - return group[pos_to_offset(bitmap, i)]; - } - - // Syntactic sugar. It's easy to return a const reference. To - // return a non-const reference, we need to use the assigner adaptor. - const_reference operator[](size_type i) const { - return get(i); - } - - element_adaptor operator[](size_type i) { - return element_adaptor(this, i); - } - - private: - // Create space at group[offset], assuming value_type has trivial - // copy constructor and destructor, and the allocator_type is - // the default libc_allocator_with_alloc. (Really, we want it to have - // "trivial move", because that's what realloc and memmove both do. - // But there's no way to capture that using type_traits, so we - // pretend that move(x, y) is equivalent to "x.~T(); new(x) T(y);" - // which is pretty much correct, if a bit conservative.) - void set_aux(size_type offset, base::true_type) { - group = settings.realloc_or_die(group, settings.num_buckets+1); - // This is equivalent to memmove(), but faster on my Intel P4, - // at least with gcc4.1 -O2 / glibc 2.3.6. - for (size_type i = settings.num_buckets; i > offset; --i) +// Copyright (c) 2005, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// +// +// A sparsetable is a random container that implements a sparse array, +// that is, an array that uses very little memory to store unassigned +// indices (in this case, between 1-2 bits per unassigned index). For +// instance, if you allocate an array of size 5 and assign a[2] = <big +// struct>, then a[2] will take up a lot of memory but a[0], a[1], +// a[3], and a[4] will not. Array elements that have a value are +// called "assigned". Array elements that have no value yet, or have +// had their value cleared using erase() or clear(), are called +// "unassigned". +// +// Unassigned values seem to have the default value of T (see below). +// Nevertheless, there is a difference between an unassigned index and +// one explicitly assigned the value of T(). The latter is considered +// assigned. +// +// Access to an array element is constant time, as is insertion and +// deletion. Insertion and deletion may be fairly slow, however: +// because of this container's memory economy, each insert and delete +// causes a memory reallocation. +// +// NOTE: You should not test(), get(), or set() any index that is +// greater than sparsetable.size(). If you need to do that, call +// resize() first. +// +// --- Template parameters +// PARAMETER DESCRIPTION DEFAULT +// T The value of the array: the type of -- +// object that is stored in the array. +// +// GROUP_SIZE How large each "group" in the table 48 +// is (see below). Larger values use +// a little less memory but cause most +// operations to be a little slower +// +// Alloc: Allocator to use to allocate memory. libc_allocator_with_realloc +// +// --- Model of +// Random Access Container +// +// --- Type requirements +// T must be Copy Constructible. It need not be Assignable. +// +// --- Public base classes +// None. +// +// --- Members +// Type members +// +// MEMBER WHERE DEFINED DESCRIPTION +// value_type container The type of object, T, stored in the array +// allocator_type container Allocator to use +// pointer container Pointer to p +// const_pointer container Const pointer to p +// reference container Reference to t +// const_reference container Const reference to t +// size_type container An unsigned integral type +// difference_type container A signed integral type +// iterator [*] container Iterator used to iterate over a sparsetable +// const_iterator container Const iterator used to iterate over a table +// reverse_iterator reversible Iterator used to iterate backwards over +// container a sparsetable +// const_reverse_iterator reversible container Guess +// nonempty_iterator [+] sparsetable Iterates over assigned +// array elements only +// const_nonempty_iterator sparsetable Iterates over assigned +// array elements only +// reverse_nonempty_iterator sparsetable Iterates backwards over +// assigned array elements only +// const_reverse_nonempty_iterator sparsetable Iterates backwards over +// assigned array elements only +// +// [*] All iterators are const in a sparsetable (though nonempty_iterators +// may not be). Use get() and set() to assign values, not iterators. +// +// [+] iterators are random-access iterators. nonempty_iterators are +// bidirectional iterators. + +// Iterator members +// MEMBER WHERE DEFINED DESCRIPTION +// +// iterator begin() container An iterator to the beginning of the table +// iterator end() container An iterator to the end of the table +// const_iterator container A const_iterator pointing to the +// begin() const beginning of a sparsetable +// const_iterator container A const_iterator pointing to the +// end() const end of a sparsetable +// +// reverse_iterator reversable Points to beginning of a reversed +// rbegin() container sparsetable +// reverse_iterator reversable Points to end of a reversed table +// rend() container +// const_reverse_iterator reversable Points to beginning of a +// rbegin() const container reversed sparsetable +// const_reverse_iterator reversable Points to end of a reversed table +// rend() const container +// +// nonempty_iterator sparsetable Points to first assigned element +// begin() of a sparsetable +// nonempty_iterator sparsetable Points past last assigned element +// end() of a sparsetable +// const_nonempty_iterator sparsetable Points to first assigned element +// begin() const of a sparsetable +// const_nonempty_iterator sparsetable Points past last assigned element +// end() const of a sparsetable +// +// reverse_nonempty_iterator sparsetable Points to first assigned element +// begin() of a reversed sparsetable +// reverse_nonempty_iterator sparsetable Points past last assigned element +// end() of a reversed sparsetable +// const_reverse_nonempty_iterator sparsetable Points to first assigned +// begin() const elt of a reversed sparsetable +// const_reverse_nonempty_iterator sparsetable Points past last assigned +// end() const elt of a reversed sparsetable +// +// +// Other members +// MEMBER WHERE DEFINED DESCRIPTION +// sparsetable() sparsetable A table of size 0; must resize() +// before using. +// sparsetable(size_type size) sparsetable A table of size size. All +// indices are unassigned. +// sparsetable( +// const sparsetable &tbl) sparsetable Copy constructor +// ~sparsetable() sparsetable The destructor +// sparsetable &operator=( sparsetable The assignment operator +// const sparsetable &tbl) +// +// void resize(size_type size) sparsetable Grow or shrink a table to +// have size indices [*] +// +// void swap(sparsetable &x) sparsetable Swap two sparsetables +// void swap(sparsetable &x, sparsetable Swap two sparsetables +// sparsetable &y) (global, not member, function) +// +// size_type size() const sparsetable Number of "buckets" in the table +// size_type max_size() const sparsetable Max allowed size of a sparsetable +// bool empty() const sparsetable true if size() == 0 +// size_type num_nonempty() const sparsetable Number of assigned "buckets" +// +// const_reference get( sparsetable Value at index i, or default +// size_type i) const value if i is unassigned +// const_reference operator[]( sparsetable Identical to get(i) [+] +// difference_type i) const +// reference set(size_type i, sparsetable Set element at index i to +// const_reference val) be a copy of val +// bool test(size_type i) sparsetable True if element at index i +// const has been assigned to +// bool test(iterator pos) sparsetable True if element pointed to +// const by pos has been assigned to +// void erase(iterator pos) sparsetable Set element pointed to by +// pos to be unassigned [!] +// void erase(size_type i) sparsetable Set element i to be unassigned +// void erase(iterator start, sparsetable Erases all elements between +// iterator end) start and end +// void clear() sparsetable Erases all elements in the table +// +// I/O versions exist for both FILE* and for File* (Google2-style files): +// bool write_metadata(FILE *fp) sparsetable Writes a sparsetable to the +// bool write_metadata(File *fp) given file. true if write +// completes successfully +// bool read_metadata(FILE *fp) sparsetable Replaces sparsetable with +// bool read_metadata(File *fp) version read from fp. true +// if read completes sucessfully +// bool write_nopointer_data(FILE *fp) Read/write the data stored in +// bool read_nopointer_data(FILE*fp) the table, if it's simple +// +// bool operator==( forward Tests two tables for equality. +// const sparsetable &t1, container This is a global function, +// const sparsetable &t2) not a member function. +// bool operator<( forward Lexicographical comparison. +// const sparsetable &t1, container This is a global function, +// const sparsetable &t2) not a member function. +// +// [*] If you shrink a sparsetable using resize(), assigned elements +// past the end of the table are removed using erase(). If you grow +// a sparsetable, new unassigned indices are created. +// +// [+] Note that operator[] returns a const reference. You must use +// set() to change the value of a table element. +// +// [!] Unassignment also calls the destructor. +// +// Iterators are invalidated whenever an item is inserted or +// deleted (ie set() or erase() is used) or when the size of +// the table changes (ie resize() or clear() is used). +// +// See doc/sparsetable.html for more information about how to use this class. + +// Note: this uses STL style for naming, rather than Google naming. +// That's because this is an STL-y container + +#ifndef UTIL_GTL_SPARSETABLE_H_ +#define UTIL_GTL_SPARSETABLE_H_ + +#include <sparsehash/internal/sparseconfig.h> +#include <stdlib.h> // for malloc/free +#include <stdio.h> // to read/write tables +#include <string.h> // for memcpy +#ifdef HAVE_STDINT_H +#include <stdint.h> // the normal place uint16_t is defined +#endif +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> // the normal place u_int16_t is defined +#endif +#ifdef HAVE_INTTYPES_H +#include <inttypes.h> // a third place for uint16_t or u_int16_t +#endif +#include <assert.h> // for bounds checking +#include <iterator> // to define reverse_iterator for me +#include <algorithm> // equal, lexicographical_compare, swap,... +#include <memory> // uninitialized_copy, uninitialized_fill +#include <vector> // a sparsetable is a vector of groups +#include <sparsehash/type_traits.h> +#include <sparsehash/internal/hashtable-common.h> +#include <sparsehash/internal/libc_allocator_with_realloc.h> + +// A lot of work to get a type that's guaranteed to be 16 bits... +#ifndef HAVE_U_INT16_T +# if defined HAVE_UINT16_T + typedef uint16_t u_int16_t; // true on solaris, possibly other C99 libc's +# elif defined HAVE___UINT16 + typedef __int16 int16_t; // true on vc++7 + typedef unsigned __int16 u_int16_t; +# else + // Cannot find a 16-bit integer type. Hoping for the best with "short"... + typedef short int int16_t; + typedef unsigned short int u_int16_t; +# endif +#endif + +_START_GOOGLE_NAMESPACE_ + +namespace base { // just to make google->opensource transition easier +using GOOGLE_NAMESPACE::true_type; +using GOOGLE_NAMESPACE::false_type; +using GOOGLE_NAMESPACE::integral_constant; +using GOOGLE_NAMESPACE::has_trivial_copy; +using GOOGLE_NAMESPACE::has_trivial_destructor; +using GOOGLE_NAMESPACE::is_same; +} + + +// The smaller this is, the faster lookup is (because the group bitmap is +// smaller) and the faster insert is, because there's less to move. +// On the other hand, there are more groups. Since group::size_type is +// a short, this number should be of the form 32*x + 16 to avoid waste. +static const u_int16_t DEFAULT_SPARSEGROUP_SIZE = 48; // fits in 1.5 words + + +// Our iterator as simple as iterators can be: basically it's just +// the index into our table. Dereference, the only complicated +// thing, we punt to the table class. This just goes to show how +// much machinery STL requires to do even the most trivial tasks. +// +// A NOTE ON ASSIGNING: +// A sparse table does not actually allocate memory for entries +// that are not filled. Because of this, it becomes complicated +// to have a non-const iterator: we don't know, if the iterator points +// to a not-filled bucket, whether you plan to fill it with something +// or whether you plan to read its value (in which case you'll get +// the default bucket value). Therefore, while we can define const +// operations in a pretty 'normal' way, for non-const operations, we +// define something that returns a helper object with operator= and +// operator& that allocate a bucket lazily. We use this for table[] +// and also for regular table iterators. + +template <class tabletype> +class table_element_adaptor { + public: + typedef typename tabletype::value_type value_type; + typedef typename tabletype::size_type size_type; + typedef typename tabletype::reference reference; + typedef typename tabletype::pointer pointer; + + table_element_adaptor(tabletype *tbl, size_type p) + : table(tbl), pos(p) { } + table_element_adaptor& operator= (const value_type &val) { + table->set(pos, val); + return *this; + } + operator value_type() { return table->get(pos); } // we look like a value + pointer operator& () { return &table->mutating_get(pos); } + + private: + tabletype* table; + size_type pos; +}; + +// Our iterator as simple as iterators can be: basically it's just +// the index into our table. Dereference, the only complicated +// thing, we punt to the table class. This just goes to show how +// much machinery STL requires to do even the most trivial tasks. +// +// By templatizing over tabletype, we have one iterator type which +// we can use for both sparsetables and sparsebins. In fact it +// works on any class that allows size() and operator[] (eg vector), +// as long as it does the standard STL typedefs too (eg value_type). + +template <class tabletype> +class table_iterator { + public: + typedef table_iterator iterator; + + typedef std::random_access_iterator_tag iterator_category; + typedef typename tabletype::value_type value_type; + typedef typename tabletype::difference_type difference_type; + typedef typename tabletype::size_type size_type; + typedef table_element_adaptor<tabletype> reference; + typedef table_element_adaptor<tabletype>* pointer; + + // The "real" constructor + table_iterator(tabletype *tbl, size_type p) + : table(tbl), pos(p) { } + // The default constructor, used when I define vars of type table::iterator + table_iterator() : table(NULL), pos(0) { } + // The copy constructor, for when I say table::iterator foo = tbl.begin() + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // The main thing our iterator does is dereference. If the table entry + // we point to is empty, we return the default value type. + // This is the big different function from the const iterator. + reference operator*() { + return table_element_adaptor<tabletype>(table, pos); + } + pointer operator->() { return &(operator*()); } + + // Helper function to assert things are ok; eg pos is still in range + void check() const { + assert(table); + assert(pos <= table->size()); + } + + // Arithmetic: we just do arithmetic on pos. We don't even need to + // do bounds checking, since STL doesn't consider that its job. :-) + iterator& operator+=(size_type t) { pos += t; check(); return *this; } + iterator& operator-=(size_type t) { pos -= t; check(); return *this; } + iterator& operator++() { ++pos; check(); return *this; } + iterator& operator--() { --pos; check(); return *this; } + iterator operator++(int) { iterator tmp(*this); // for x++ + ++pos; check(); return tmp; } + iterator operator--(int) { iterator tmp(*this); // for x-- + --pos; check(); return tmp; } + iterator operator+(difference_type i) const { iterator tmp(*this); + tmp += i; return tmp; } + iterator operator-(difference_type i) const { iterator tmp(*this); + tmp -= i; return tmp; } + difference_type operator-(iterator it) const { // for "x = it2 - it" + assert(table == it.table); + return pos - it.pos; + } + reference operator[](difference_type n) const { + return *(*this + n); // simple though not totally efficient + } + + // Comparisons. + bool operator==(const iterator& it) const { + return table == it.table && pos == it.pos; + } + bool operator<(const iterator& it) const { + assert(table == it.table); // life is bad bad bad otherwise + return pos < it.pos; + } + bool operator!=(const iterator& it) const { return !(*this == it); } + bool operator<=(const iterator& it) const { return !(it < *this); } + bool operator>(const iterator& it) const { return it < *this; } + bool operator>=(const iterator& it) const { return !(*this < it); } + + // Here's the info we actually need to be an iterator + tabletype *table; // so we can dereference and bounds-check + size_type pos; // index into the table +}; + +// support for "3 + iterator" has to be defined outside the class, alas +template<class T> +table_iterator<T> operator+(typename table_iterator<T>::difference_type i, + table_iterator<T> it) { + return it + i; // so people can say it2 = 3 + it +} + +template <class tabletype> +class const_table_iterator { + public: + typedef table_iterator<tabletype> iterator; + typedef const_table_iterator const_iterator; + + typedef std::random_access_iterator_tag iterator_category; + typedef typename tabletype::value_type value_type; + typedef typename tabletype::difference_type difference_type; + typedef typename tabletype::size_type size_type; + typedef typename tabletype::const_reference reference; // we're const-only + typedef typename tabletype::const_pointer pointer; + + // The "real" constructor + const_table_iterator(const tabletype *tbl, size_type p) + : table(tbl), pos(p) { } + // The default constructor, used when I define vars of type table::iterator + const_table_iterator() : table(NULL), pos(0) { } + // The copy constructor, for when I say table::iterator foo = tbl.begin() + // Also converts normal iterators to const iterators + const_table_iterator(const iterator &from) + : table(from.table), pos(from.pos) { } + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // The main thing our iterator does is dereference. If the table entry + // we point to is empty, we return the default value type. + reference operator*() const { return (*table)[pos]; } + pointer operator->() const { return &(operator*()); } + + // Helper function to assert things are ok; eg pos is still in range + void check() const { + assert(table); + assert(pos <= table->size()); + } + + // Arithmetic: we just do arithmetic on pos. We don't even need to + // do bounds checking, since STL doesn't consider that its job. :-) + const_iterator& operator+=(size_type t) { pos += t; check(); return *this; } + const_iterator& operator-=(size_type t) { pos -= t; check(); return *this; } + const_iterator& operator++() { ++pos; check(); return *this; } + const_iterator& operator--() { --pos; check(); return *this; } + const_iterator operator++(int) { const_iterator tmp(*this); // for x++ + ++pos; check(); return tmp; } + const_iterator operator--(int) { const_iterator tmp(*this); // for x-- + --pos; check(); return tmp; } + const_iterator operator+(difference_type i) const { const_iterator tmp(*this); + tmp += i; return tmp; } + const_iterator operator-(difference_type i) const { const_iterator tmp(*this); + tmp -= i; return tmp; } + difference_type operator-(const_iterator it) const { // for "x = it2 - it" + assert(table == it.table); + return pos - it.pos; + } + reference operator[](difference_type n) const { + return *(*this + n); // simple though not totally efficient + } + + // Comparisons. + bool operator==(const const_iterator& it) const { + return table == it.table && pos == it.pos; + } + bool operator<(const const_iterator& it) const { + assert(table == it.table); // life is bad bad bad otherwise + return pos < it.pos; + } + bool operator!=(const const_iterator& it) const { return !(*this == it); } + bool operator<=(const const_iterator& it) const { return !(it < *this); } + bool operator>(const const_iterator& it) const { return it < *this; } + bool operator>=(const const_iterator& it) const { return !(*this < it); } + + // Here's the info we actually need to be an iterator + const tabletype *table; // so we can dereference and bounds-check + size_type pos; // index into the table +}; + +// support for "3 + iterator" has to be defined outside the class, alas +template<class T> +const_table_iterator<T> operator+(typename + const_table_iterator<T>::difference_type i, + const_table_iterator<T> it) { + return it + i; // so people can say it2 = 3 + it +} + + +// --------------------------------------------------------------------------- + + +/* +// This is a 2-D iterator. You specify a begin and end over a list +// of *containers*. We iterate over each container by iterating over +// it. It's actually simple: +// VECTOR.begin() VECTOR[0].begin() --------> VECTOR[0].end() ---, +// | ________________________________________________/ +// | \_> VECTOR[1].begin() --------> VECTOR[1].end() -, +// | ___________________________________________________/ +// v \_> ...... +// VECTOR.end() +// +// It's impossible to do random access on one of these things in constant +// time, so it's just a bidirectional iterator. +// +// Unfortunately, because we need to use this for a non-empty iterator, +// we use nonempty_begin() and nonempty_end() instead of begin() and end() +// (though only going across, not down). +*/ + +#define TWOD_BEGIN_ nonempty_begin +#define TWOD_END_ nonempty_end +#define TWOD_ITER_ nonempty_iterator +#define TWOD_CONST_ITER_ const_nonempty_iterator + +template <class containertype> +class two_d_iterator { + public: + typedef two_d_iterator iterator; + + typedef std::bidirectional_iterator_tag iterator_category; + // apparently some versions of VC++ have trouble with two ::'s in a typename + typedef typename containertype::value_type _tmp_vt; + typedef typename _tmp_vt::value_type value_type; + typedef typename _tmp_vt::difference_type difference_type; + typedef typename _tmp_vt::reference reference; + typedef typename _tmp_vt::pointer pointer; + + // The "real" constructor. begin and end specify how many rows we have + // (in the diagram above); we always iterate over each row completely. + two_d_iterator(typename containertype::iterator begin, + typename containertype::iterator end, + typename containertype::iterator curr) + : row_begin(begin), row_end(end), row_current(curr), col_current() { + if ( row_current != row_end ) { + col_current = row_current->TWOD_BEGIN_(); + advance_past_end(); // in case cur->begin() == cur->end() + } + } + // If you want to start at an arbitrary place, you can, I guess + two_d_iterator(typename containertype::iterator begin, + typename containertype::iterator end, + typename containertype::iterator curr, + typename containertype::value_type::TWOD_ITER_ col) + : row_begin(begin), row_end(end), row_current(curr), col_current(col) { + advance_past_end(); // in case cur->begin() == cur->end() + } + // The default constructor, used when I define vars of type table::iterator + two_d_iterator() : row_begin(), row_end(), row_current(), col_current() { } + // The default destructor is fine; we don't define one + // The default operator= is fine; we don't define one + + // Happy dereferencer + reference operator*() const { return *col_current; } + pointer operator->() const { return &(operator*()); } + + // Arithmetic: we just do arithmetic on pos. We don't even need to + // do bounds checking, since STL doesn't consider that its job. :-) + // NOTE: this is not amortized constant time! What do we do about it? + void advance_past_end() { // used when col_current points to end() + while ( col_current == row_current->TWOD_END_() ) { // end of current row + ++row_current; // go to beginning of next + if ( row_current != row_end ) // col is irrelevant at end + col_current = row_current->TWOD_BEGIN_(); + else + break; // don't go past row_end + } + } + + iterator& operator++() { + assert(row_current != row_end); // how to ++ from there? + ++col_current; + advance_past_end(); // in case col_current is at end() + return *this; + } + iterator& operator--() { + while ( row_current == row_end || + col_current == row_current->TWOD_BEGIN_() ) { + assert(row_current != row_begin); + --row_current; + col_current = row_current->TWOD_END_(); // this is 1 too far + } + --col_current; + return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + iterator operator--(int) { iterator tmp(*this); --*this; return tmp; } + + + // Comparisons. + bool operator==(const iterator& it) const { + return ( row_begin == it.row_begin && + row_end == it.row_end && + row_current == it.row_current && + (row_current == row_end || col_current == it.col_current) ); + } + bool operator!=(const iterator& it) const { return !(*this == it); } + + + // Here's the info we actually need to be an iterator + // These need to be public so we convert from iterator to const_iterator + typename containertype::iterator row_begin, row_end, row_current; + typename containertype::value_type::TWOD_ITER_ col_current; +}; + +// The same thing again, but this time const. :-( +template <class containertype> +class const_two_d_iterator { + public: + typedef const_two_d_iterator iterator; + + typedef std::bidirectional_iterator_tag iterator_category; + // apparently some versions of VC++ have trouble with two ::'s in a typename + typedef typename containertype::value_type _tmp_vt; + typedef typename _tmp_vt::value_type value_type; + typedef typename _tmp_vt::difference_type difference_type; + typedef typename _tmp_vt::const_reference reference; + typedef typename _tmp_vt::const_pointer pointer; + + const_two_d_iterator(typename containertype::const_iterator begin, + typename containertype::const_iterator end, + typename containertype::const_iterator curr) + : row_begin(begin), row_end(end), row_current(curr), col_current() { + if ( curr != end ) { + col_current = curr->TWOD_BEGIN_(); + advance_past_end(); // in case cur->begin() == cur->end() + } + } + const_two_d_iterator(typename containertype::const_iterator begin, + typename containertype::const_iterator end, + typename containertype::const_iterator curr, + typename containertype::value_type::TWOD_CONST_ITER_ col) + : row_begin(begin), row_end(end), row_current(curr), col_current(col) { + advance_past_end(); // in case cur->begin() == cur->end() + } + const_two_d_iterator() + : row_begin(), row_end(), row_current(), col_current() { + } + // Need this explicitly so we can convert normal iterators to const iterators + const_two_d_iterator(const two_d_iterator<containertype>& it) : + row_begin(it.row_begin), row_end(it.row_end), row_current(it.row_current), + col_current(it.col_current) { } + + typename containertype::const_iterator row_begin, row_end, row_current; + typename containertype::value_type::TWOD_CONST_ITER_ col_current; + + + // EVERYTHING FROM HERE DOWN IS THE SAME AS THE NON-CONST ITERATOR + reference operator*() const { return *col_current; } + pointer operator->() const { return &(operator*()); } + + void advance_past_end() { // used when col_current points to end() + while ( col_current == row_current->TWOD_END_() ) { // end of current row + ++row_current; // go to beginning of next + if ( row_current != row_end ) // col is irrelevant at end + col_current = row_current->TWOD_BEGIN_(); + else + break; // don't go past row_end + } + } + iterator& operator++() { + assert(row_current != row_end); // how to ++ from there? + ++col_current; + advance_past_end(); // in case col_current is at end() + return *this; + } + iterator& operator--() { + while ( row_current == row_end || + col_current == row_current->TWOD_BEGIN_() ) { + assert(row_current != row_begin); + --row_current; + col_current = row_current->TWOD_END_(); // this is 1 too far + } + --col_current; + return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + iterator operator--(int) { iterator tmp(*this); --*this; return tmp; } + + bool operator==(const iterator& it) const { + return ( row_begin == it.row_begin && + row_end == it.row_end && + row_current == it.row_current && + (row_current == row_end || col_current == it.col_current) ); + } + bool operator!=(const iterator& it) const { return !(*this == it); } +}; + +// We provide yet another version, to be as frugal with memory as +// possible. This one frees each block of memory as it finishes +// iterating over it. By the end, the entire table is freed. +// For understandable reasons, you can only iterate over it once, +// which is why it's an input iterator +template <class containertype> +class destructive_two_d_iterator { + public: + typedef destructive_two_d_iterator iterator; + + typedef std::input_iterator_tag iterator_category; + // apparently some versions of VC++ have trouble with two ::'s in a typename + typedef typename containertype::value_type _tmp_vt; + typedef typename _tmp_vt::value_type value_type; + typedef typename _tmp_vt::difference_type difference_type; + typedef typename _tmp_vt::reference reference; + typedef typename _tmp_vt::pointer pointer; + + destructive_two_d_iterator(typename containertype::iterator begin, + typename containertype::iterator end, + typename containertype::iterator curr) + : row_begin(begin), row_end(end), row_current(curr), col_current() { + if ( curr != end ) { + col_current = curr->TWOD_BEGIN_(); + advance_past_end(); // in case cur->begin() == cur->end() + } + } + destructive_two_d_iterator(typename containertype::iterator begin, + typename containertype::iterator end, + typename containertype::iterator curr, + typename containertype::value_type::TWOD_ITER_ col) + : row_begin(begin), row_end(end), row_current(curr), col_current(col) { + advance_past_end(); // in case cur->begin() == cur->end() + } + destructive_two_d_iterator() + : row_begin(), row_end(), row_current(), col_current() { + } + + typename containertype::iterator row_begin, row_end, row_current; + typename containertype::value_type::TWOD_ITER_ col_current; + + // This is the part that destroys + void advance_past_end() { // used when col_current points to end() + while ( col_current == row_current->TWOD_END_() ) { // end of current row + row_current->clear(); // the destructive part + // It would be nice if we could decrement sparsetable->num_buckets here + ++row_current; // go to beginning of next + if ( row_current != row_end ) // col is irrelevant at end + col_current = row_current->TWOD_BEGIN_(); + else + break; // don't go past row_end + } + } + + // EVERYTHING FROM HERE DOWN IS THE SAME AS THE REGULAR ITERATOR + reference operator*() const { return *col_current; } + pointer operator->() const { return &(operator*()); } + + iterator& operator++() { + assert(row_current != row_end); // how to ++ from there? + ++col_current; + advance_past_end(); // in case col_current is at end() + return *this; + } + iterator operator++(int) { iterator tmp(*this); ++*this; return tmp; } + + bool operator==(const iterator& it) const { + return ( row_begin == it.row_begin && + row_end == it.row_end && + row_current == it.row_current && + (row_current == row_end || col_current == it.col_current) ); + } + bool operator!=(const iterator& it) const { return !(*this == it); } +}; + +#undef TWOD_BEGIN_ +#undef TWOD_END_ +#undef TWOD_ITER_ +#undef TWOD_CONST_ITER_ + + + + +// SPARSE-TABLE +// ------------ +// The idea is that a table with (logically) t buckets is divided +// into t/M *groups* of M buckets each. (M is a constant set in +// GROUP_SIZE for efficiency.) Each group is stored sparsely. +// Thus, inserting into the table causes some array to grow, which is +// slow but still constant time. Lookup involves doing a +// logical-position-to-sparse-position lookup, which is also slow but +// constant time. The larger M is, the slower these operations are +// but the less overhead (slightly). +// +// To store the sparse array, we store a bitmap B, where B[i] = 1 iff +// bucket i is non-empty. Then to look up bucket i we really look up +// array[# of 1s before i in B]. This is constant time for fixed M. +// +// Terminology: the position of an item in the overall table (from +// 1 .. t) is called its "location." The logical position in a group +// (from 1 .. M ) is called its "position." The actual location in +// the array (from 1 .. # of non-empty buckets in the group) is +// called its "offset." + +template <class T, u_int16_t GROUP_SIZE, class Alloc> +class sparsegroup { + private: + typedef typename Alloc::template rebind<T>::other value_alloc_type; + + public: + // Basic types + typedef T value_type; + typedef Alloc allocator_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::const_reference const_reference; + typedef typename value_alloc_type::pointer pointer; + typedef typename value_alloc_type::const_pointer const_pointer; + + typedef table_iterator<sparsegroup<T, GROUP_SIZE, Alloc> > iterator; + typedef const_table_iterator<sparsegroup<T, GROUP_SIZE, Alloc> > + const_iterator; + typedef table_element_adaptor<sparsegroup<T, GROUP_SIZE, Alloc> > + element_adaptor; + typedef u_int16_t size_type; // max # of buckets + typedef int16_t difference_type; + typedef std::reverse_iterator<const_iterator> const_reverse_iterator; + typedef std::reverse_iterator<iterator> reverse_iterator; // from iterator.h + + // These are our special iterators, that go over non-empty buckets in a + // group. These aren't const-only because you can change non-empty bcks. + typedef pointer nonempty_iterator; + typedef const_pointer const_nonempty_iterator; + typedef std::reverse_iterator<nonempty_iterator> reverse_nonempty_iterator; + typedef std::reverse_iterator<const_nonempty_iterator> const_reverse_nonempty_iterator; + + // Iterator functions + iterator begin() { return iterator(this, 0); } + const_iterator begin() const { return const_iterator(this, 0); } + iterator end() { return iterator(this, size()); } + const_iterator end() const { return const_iterator(this, size()); } + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } + + // We'll have versions for our special non-empty iterator too + nonempty_iterator nonempty_begin() { return group; } + const_nonempty_iterator nonempty_begin() const { return group; } + nonempty_iterator nonempty_end() { + return group + settings.num_buckets; + } + const_nonempty_iterator nonempty_end() const { + return group + settings.num_buckets; + } + reverse_nonempty_iterator nonempty_rbegin() { + return reverse_nonempty_iterator(nonempty_end()); + } + const_reverse_nonempty_iterator nonempty_rbegin() const { + return const_reverse_nonempty_iterator(nonempty_end()); + } + reverse_nonempty_iterator nonempty_rend() { + return reverse_nonempty_iterator(nonempty_begin()); + } + const_reverse_nonempty_iterator nonempty_rend() const { + return const_reverse_nonempty_iterator(nonempty_begin()); + } + + + // This gives us the "default" value to return for an empty bucket. + // We just use the default constructor on T, the template type + const_reference default_value() const { + static value_type defaultval = value_type(); + return defaultval; + } + + + private: + // We need to do all this bit manipulation, of course. ick + static size_type charbit(size_type i) { return i >> 3; } + static size_type modbit(size_type i) { return 1 << (i&7); } + int bmtest(size_type i) const { return bitmap[charbit(i)] & modbit(i); } + void bmset(size_type i) { bitmap[charbit(i)] |= modbit(i); } + void bmclear(size_type i) { bitmap[charbit(i)] &= ~modbit(i); } + + pointer allocate_group(size_type n) { + pointer retval = settings.allocate(n); + if (retval == NULL) { + // We really should use PRIuS here, but I don't want to have to add + // a whole new configure option, with concomitant macro namespace + // pollution, just to print this (unlikely) error message. So I cast. + fprintf(stderr, "sparsehash FATAL ERROR: failed to allocate %lu groups\n", + static_cast<unsigned long>(n)); + exit(1); + } + return retval; + } + + void free_group() { + if (!group) return; + pointer end_it = group + settings.num_buckets; + for (pointer p = group; p != end_it; ++p) + p->~value_type(); + settings.deallocate(group, settings.num_buckets); + group = NULL; + } + + static size_type bits_in_char(unsigned char c) { + // We could make these ints. The tradeoff is size (eg does it overwhelm + // the cache?) vs efficiency in referencing sub-word-sized array elements. + static const char bits_in[256] = { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8, + }; + return bits_in[c]; + } + + public: // get_iter() in sparsetable needs it + // We need a small function that tells us how many set bits there are + // in positions 0..i-1 of the bitmap. It uses a big table. + // We make it static so templates don't allocate lots of these tables. + // There are lots of ways to do this calculation (called 'popcount'). + // The 8-bit table lookup is one of the fastest, though this + // implementation suffers from not doing any loop unrolling. See, eg, + // http://www.dalkescientific.com/writings/diary/archive/2008/07/03/hakmem_and_other_popcounts.html + // http://gurmeetsingh.wordpress.com/2008/08/05/fast-bit-counting-routines/ + static size_type pos_to_offset(const unsigned char *bm, size_type pos) { + size_type retval = 0; + + // [Note: condition pos > 8 is an optimization; convince yourself we + // give exactly the same result as if we had pos >= 8 here instead.] + for ( ; pos > 8; pos -= 8 ) // bm[0..pos/8-1] + retval += bits_in_char(*bm++); // chars we want *all* bits in + return retval + bits_in_char(*bm & ((1 << pos)-1)); // char including pos + } + + size_type pos_to_offset(size_type pos) const { // not static but still const + return pos_to_offset(bitmap, pos); + } + + // Returns the (logical) position in the bm[] array, i, such that + // bm[i] is the offset-th set bit in the array. It is the inverse + // of pos_to_offset. get_pos() uses this function to find the index + // of an nonempty_iterator in the table. Bit-twiddling from + // http://hackersdelight.org/basics.pdf + static size_type offset_to_pos(const unsigned char *bm, size_type offset) { + size_type retval = 0; + // This is sizeof(this->bitmap). + const size_type group_size = (GROUP_SIZE-1) / 8 + 1; + for (size_type i = 0; i < group_size; i++) { // forward scan + const size_type pop_count = bits_in_char(*bm); + if (pop_count > offset) { + unsigned char last_bm = *bm; + for (; offset > 0; offset--) { + last_bm &= (last_bm-1); // remove right-most set bit + } + // Clear all bits to the left of the rightmost bit (the &), + // and then clear the rightmost bit but set all bits to the + // right of it (the -1). + last_bm = (last_bm & -last_bm) - 1; + retval += bits_in_char(last_bm); + return retval; + } + offset -= pop_count; + retval += 8; + bm++; + } + return retval; + } + + size_type offset_to_pos(size_type offset) const { + return offset_to_pos(bitmap, offset); + } + + + public: + // Constructors -- default and copy -- and destructor + explicit sparsegroup(allocator_type& a) : + group(0), settings(alloc_impl<value_alloc_type>(a)) { + memset(bitmap, 0, sizeof(bitmap)); + } + sparsegroup(const sparsegroup& x) : group(0), settings(x.settings) { + if ( settings.num_buckets ) { + group = allocate_group(x.settings.num_buckets); + std::uninitialized_copy(x.group, x.group + x.settings.num_buckets, group); + } + memcpy(bitmap, x.bitmap, sizeof(bitmap)); + } + ~sparsegroup() { free_group(); } + + // Operator= is just like the copy constructor, I guess + // TODO(austern): Make this exception safe. Handle exceptions in value_type's + // copy constructor. + sparsegroup &operator=(const sparsegroup& x) { + if ( &x == this ) return *this; // x = x + if ( x.settings.num_buckets == 0 ) { + free_group(); + } else { + pointer p = allocate_group(x.settings.num_buckets); + std::uninitialized_copy(x.group, x.group + x.settings.num_buckets, p); + free_group(); + group = p; + } + memcpy(bitmap, x.bitmap, sizeof(bitmap)); + settings.num_buckets = x.settings.num_buckets; + return *this; + } + + // Many STL algorithms use swap instead of copy constructors + void swap(sparsegroup& x) { + std::swap(group, x.group); // defined in <algorithm> + for ( int i = 0; i < sizeof(bitmap) / sizeof(*bitmap); ++i ) + std::swap(bitmap[i], x.bitmap[i]); // swap not defined on arrays + std::swap(settings.num_buckets, x.settings.num_buckets); + // we purposefully don't swap the allocator, which may not be swap-able + } + + // It's always nice to be able to clear a table without deallocating it + void clear() { + free_group(); + memset(bitmap, 0, sizeof(bitmap)); + settings.num_buckets = 0; + } + + // Functions that tell you about size. Alas, these aren't so useful + // because our table is always fixed size. + size_type size() const { return GROUP_SIZE; } + size_type max_size() const { return GROUP_SIZE; } + bool empty() const { return false; } + // We also may want to know how many *used* buckets there are + size_type num_nonempty() const { return settings.num_buckets; } + + + // get()/set() are explicitly const/non-const. You can use [] if + // you want something that can be either (potentially more expensive). + const_reference get(size_type i) const { + if ( bmtest(i) ) // bucket i is occupied + return group[pos_to_offset(bitmap, i)]; + else + return default_value(); // return the default reference + } + + // TODO(csilvers): make protected + friend + // This is used by sparse_hashtable to get an element from the table + // when we know it exists. + const_reference unsafe_get(size_type i) const { + assert(bmtest(i)); + return group[pos_to_offset(bitmap, i)]; + } + + // TODO(csilvers): make protected + friend + reference mutating_get(size_type i) { // fills bucket i before getting + if ( !bmtest(i) ) + set(i, default_value()); + return group[pos_to_offset(bitmap, i)]; + } + + // Syntactic sugar. It's easy to return a const reference. To + // return a non-const reference, we need to use the assigner adaptor. + const_reference operator[](size_type i) const { + return get(i); + } + + element_adaptor operator[](size_type i) { + return element_adaptor(this, i); + } + + private: + // Create space at group[offset], assuming value_type has trivial + // copy constructor and destructor, and the allocator_type is + // the default libc_allocator_with_alloc. (Really, we want it to have + // "trivial move", because that's what realloc and memmove both do. + // But there's no way to capture that using type_traits, so we + // pretend that move(x, y) is equivalent to "x.~T(); new(x) T(y);" + // which is pretty much correct, if a bit conservative.) + void set_aux(size_type offset, base::true_type) { + group = settings.realloc_or_die(group, settings.num_buckets+1); + // This is equivalent to memmove(), but faster on my Intel P4, + // at least with gcc4.1 -O2 / glibc 2.3.6. + for (size_type i = settings.num_buckets; i > offset; --i) // cast to void* to prevent compiler warnings about writing to an object // with no trivial copy-assignment memcpy(static_cast<void*>(group + i), group + i-1, sizeof(*group)); - } - - // Create space at group[offset], without special assumptions about value_type - // and allocator_type. - void set_aux(size_type offset, base::false_type) { - // This is valid because 0 <= offset <= num_buckets - pointer p = allocate_group(settings.num_buckets + 1); - std::uninitialized_copy(group, group + offset, p); - std::uninitialized_copy(group + offset, group + settings.num_buckets, - p + offset + 1); - free_group(); - group = p; - } - - public: - // This returns a reference to the inserted item (which is a copy of val). - // TODO(austern): Make this exception safe: handle exceptions from - // value_type's copy constructor. - reference set(size_type i, const_reference val) { - size_type offset = pos_to_offset(bitmap, i); // where we'll find (or insert) - if ( bmtest(i) ) { - // Delete the old value, which we're replacing with the new one - group[offset].~value_type(); - } else { - typedef base::integral_constant<bool, - (base::has_trivial_copy<value_type>::value && - base::has_trivial_destructor<value_type>::value && - base::is_same< - allocator_type, - libc_allocator_with_realloc<value_type> >::value)> - realloc_and_memmove_ok; // we pretend mv(x,y) == "x.~T(); new(x) T(y)" - set_aux(offset, realloc_and_memmove_ok()); - ++settings.num_buckets; - bmset(i); - } - // This does the actual inserting. Since we made the array using - // malloc, we use "placement new" to just call the constructor. - new(&group[offset]) value_type(val); - return group[offset]; - } - - // We let you see if a bucket is non-empty without retrieving it - bool test(size_type i) const { - return bmtest(i) != 0; - } - bool test(iterator pos) const { - return bmtest(pos.pos) != 0; - } - - private: - // Shrink the array, assuming value_type has trivial copy - // constructor and destructor, and the allocator_type is the default - // libc_allocator_with_alloc. (Really, we want it to have "trivial - // move", because that's what realloc and memmove both do. But - // there's no way to capture that using type_traits, so we pretend - // that move(x, y) is equivalent to ""x.~T(); new(x) T(y);" - // which is pretty much correct, if a bit conservative.) - void erase_aux(size_type offset, base::true_type) { - // This isn't technically necessary, since we know we have a - // trivial destructor, but is a cheap way to get a bit more safety. - group[offset].~value_type(); - // This is equivalent to memmove(), but faster on my Intel P4, - // at lesat with gcc4.1 -O2 / glibc 2.3.6. - assert(settings.num_buckets > 0); - for (size_type i = offset; i < settings.num_buckets-1; ++i) + } + + // Create space at group[offset], without special assumptions about value_type + // and allocator_type. + void set_aux(size_type offset, base::false_type) { + // This is valid because 0 <= offset <= num_buckets + pointer p = allocate_group(settings.num_buckets + 1); + std::uninitialized_copy(group, group + offset, p); + std::uninitialized_copy(group + offset, group + settings.num_buckets, + p + offset + 1); + free_group(); + group = p; + } + + public: + // This returns a reference to the inserted item (which is a copy of val). + // TODO(austern): Make this exception safe: handle exceptions from + // value_type's copy constructor. + reference set(size_type i, const_reference val) { + size_type offset = pos_to_offset(bitmap, i); // where we'll find (or insert) + if ( bmtest(i) ) { + // Delete the old value, which we're replacing with the new one + group[offset].~value_type(); + } else { + typedef base::integral_constant<bool, + (base::has_trivial_copy<value_type>::value && + base::has_trivial_destructor<value_type>::value && + base::is_same< + allocator_type, + libc_allocator_with_realloc<value_type> >::value)> + realloc_and_memmove_ok; // we pretend mv(x,y) == "x.~T(); new(x) T(y)" + set_aux(offset, realloc_and_memmove_ok()); + ++settings.num_buckets; + bmset(i); + } + // This does the actual inserting. Since we made the array using + // malloc, we use "placement new" to just call the constructor. + new(&group[offset]) value_type(val); + return group[offset]; + } + + // We let you see if a bucket is non-empty without retrieving it + bool test(size_type i) const { + return bmtest(i) != 0; + } + bool test(iterator pos) const { + return bmtest(pos.pos) != 0; + } + + private: + // Shrink the array, assuming value_type has trivial copy + // constructor and destructor, and the allocator_type is the default + // libc_allocator_with_alloc. (Really, we want it to have "trivial + // move", because that's what realloc and memmove both do. But + // there's no way to capture that using type_traits, so we pretend + // that move(x, y) is equivalent to ""x.~T(); new(x) T(y);" + // which is pretty much correct, if a bit conservative.) + void erase_aux(size_type offset, base::true_type) { + // This isn't technically necessary, since we know we have a + // trivial destructor, but is a cheap way to get a bit more safety. + group[offset].~value_type(); + // This is equivalent to memmove(), but faster on my Intel P4, + // at lesat with gcc4.1 -O2 / glibc 2.3.6. + assert(settings.num_buckets > 0); + for (size_type i = offset; i < settings.num_buckets-1; ++i) // cast to void* to prevent compiler warnings about writing to an object // with no trivial copy-assignment // hopefully inlined! memcpy(static_cast<void*>(group + i), group + i+1, sizeof(*group)); - group = settings.realloc_or_die(group, settings.num_buckets-1); - } - - // Shrink the array, without any special assumptions about value_type and - // allocator_type. - void erase_aux(size_type offset, base::false_type) { - // This is valid because 0 <= offset < num_buckets. Note the inequality. - pointer p = allocate_group(settings.num_buckets - 1); - std::uninitialized_copy(group, group + offset, p); - std::uninitialized_copy(group + offset + 1, group + settings.num_buckets, - p + offset); - free_group(); - group = p; - } - - public: - // This takes the specified elements out of the group. This is - // "undefining", rather than "clearing". - // TODO(austern): Make this exception safe: handle exceptions from - // value_type's copy constructor. - void erase(size_type i) { - if ( bmtest(i) ) { // trivial to erase empty bucket - size_type offset = pos_to_offset(bitmap,i); // where we'll find (or insert) - if ( settings.num_buckets == 1 ) { - free_group(); - group = NULL; - } else { - typedef base::integral_constant<bool, - (base::has_trivial_copy<value_type>::value && - base::has_trivial_destructor<value_type>::value && - base::is_same< - allocator_type, - libc_allocator_with_realloc<value_type> >::value)> - realloc_and_memmove_ok; // pretend mv(x,y) == "x.~T(); new(x) T(y)" - erase_aux(offset, realloc_and_memmove_ok()); - } - --settings.num_buckets; - bmclear(i); - } - } - - void erase(iterator pos) { - erase(pos.pos); - } - - void erase(iterator start_it, iterator end_it) { - // This could be more efficient, but to do so we'd need to make - // bmclear() clear a range of indices. Doesn't seem worth it. - for ( ; start_it != end_it; ++start_it ) - erase(start_it); - } - - - // I/O - // We support reading and writing groups to disk. We don't store - // the actual array contents (which we don't know how to store), - // just the bitmap and size. Meant to be used with table I/O. - - template <typename OUTPUT> bool write_metadata(OUTPUT *fp) const { - // we explicitly set to u_int16_t - assert(sizeof(settings.num_buckets) == 2); - if ( !sparsehash_internal::write_bigendian_number(fp, settings.num_buckets, - 2) ) - return false; - if ( !sparsehash_internal::write_data(fp, bitmap, sizeof(bitmap)) ) - return false; - return true; - } - - // Reading destroys the old group contents! Returns true if all was ok. - template <typename INPUT> bool read_metadata(INPUT *fp) { - clear(); - if ( !sparsehash_internal::read_bigendian_number(fp, &settings.num_buckets, - 2) ) - return false; - if ( !sparsehash_internal::read_data(fp, bitmap, sizeof(bitmap)) ) - return false; - // We'll allocate the space, but we won't fill it: it will be - // left as uninitialized raw memory. - group = allocate_group(settings.num_buckets); - return true; - } - - // Again, only meaningful if value_type is a POD. - template <typename INPUT> bool read_nopointer_data(INPUT *fp) { - for ( nonempty_iterator it = nonempty_begin(); - it != nonempty_end(); ++it ) { - if ( !sparsehash_internal::read_data(fp, &(*it), sizeof(*it)) ) - return false; - } - return true; - } - - // If your keys and values are simple enough, we can write them - // to disk for you. "simple enough" means POD and no pointers. - // However, we don't try to normalize endianness. - template <typename OUTPUT> bool write_nopointer_data(OUTPUT *fp) const { - for ( const_nonempty_iterator it = nonempty_begin(); - it != nonempty_end(); ++it ) { - if ( !sparsehash_internal::write_data(fp, &(*it), sizeof(*it)) ) - return false; - } - return true; - } - - - // Comparisons. We only need to define == and < -- we get - // != > <= >= via relops.h (which we happily included above). - // Note the comparisons are pretty arbitrary: we compare - // values of the first index that isn't equal (using default - // value for empty buckets). - bool operator==(const sparsegroup& x) const { - return ( settings.num_buckets == x.settings.num_buckets && - memcmp(bitmap, x.bitmap, sizeof(bitmap)) == 0 && - std::equal(begin(), end(), x.begin()) ); // from <algorithm> - } - - bool operator<(const sparsegroup& x) const { // also from <algorithm> - return std::lexicographical_compare(begin(), end(), x.begin(), x.end()); - } - bool operator!=(const sparsegroup& x) const { return !(*this == x); } - bool operator<=(const sparsegroup& x) const { return !(x < *this); } - bool operator>(const sparsegroup& x) const { return x < *this; } - bool operator>=(const sparsegroup& x) const { return !(*this < x); } - - private: - template <class A> - class alloc_impl : public A { - public: - typedef typename A::pointer pointer; - typedef typename A::size_type size_type; - - // Convert a normal allocator to one that has realloc_or_die() - alloc_impl(const A& a) : A(a) { } - - // realloc_or_die should only be used when using the default - // allocator (libc_allocator_with_realloc). - pointer realloc_or_die(pointer /*ptr*/, size_type /*n*/) { - fprintf(stderr, "realloc_or_die is only supported for " - "libc_allocator_with_realloc\n"); - exit(1); - return NULL; - } - }; - - // A template specialization of alloc_impl for - // libc_allocator_with_realloc that can handle realloc_or_die. - template <class A> - class alloc_impl<libc_allocator_with_realloc<A> > - : public libc_allocator_with_realloc<A> { - public: - typedef typename libc_allocator_with_realloc<A>::pointer pointer; - typedef typename libc_allocator_with_realloc<A>::size_type size_type; - - alloc_impl(const libc_allocator_with_realloc<A>& a) - : libc_allocator_with_realloc<A>(a) { } - - pointer realloc_or_die(pointer ptr, size_type n) { - pointer retval = this->reallocate(ptr, n); - if (retval == NULL) { - fprintf(stderr, "sparsehash: FATAL ERROR: failed to reallocate " - "%lu elements for ptr %p", static_cast<unsigned long>(n), ptr); - exit(1); - } - return retval; - } - }; - - // Package allocator with num_buckets to eliminate memory needed for the - // zero-size allocator. - // If new fields are added to this class, we should add them to - // operator= and swap. - class Settings : public alloc_impl<value_alloc_type> { - public: - Settings(const alloc_impl<value_alloc_type>& a, u_int16_t n = 0) - : alloc_impl<value_alloc_type>(a), num_buckets(n) { } - Settings(const Settings& s) - : alloc_impl<value_alloc_type>(s), num_buckets(s.num_buckets) { } - - u_int16_t num_buckets; // limits GROUP_SIZE to 64K - }; - - // The actual data - pointer group; // (small) array of T's - Settings settings; // allocator and num_buckets - unsigned char bitmap[(GROUP_SIZE-1)/8 + 1]; // fancy math is so we round up -}; - -// We need a global swap as well -template <class T, u_int16_t GROUP_SIZE, class Alloc> -inline void swap(sparsegroup<T,GROUP_SIZE,Alloc> &x, - sparsegroup<T,GROUP_SIZE,Alloc> &y) { - x.swap(y); -} - -// --------------------------------------------------------------------------- - - -template <class T, u_int16_t GROUP_SIZE = DEFAULT_SPARSEGROUP_SIZE, - class Alloc = libc_allocator_with_realloc<T> > -class sparsetable { - private: - typedef typename Alloc::template rebind<T>::other value_alloc_type; - typedef typename Alloc::template rebind< - sparsegroup<T, GROUP_SIZE, value_alloc_type> >::other vector_alloc; - - public: - // Basic types - typedef T value_type; // stolen from stl_vector.h - typedef Alloc allocator_type; - typedef typename value_alloc_type::size_type size_type; - typedef typename value_alloc_type::difference_type difference_type; - typedef typename value_alloc_type::reference reference; - typedef typename value_alloc_type::const_reference const_reference; - typedef typename value_alloc_type::pointer pointer; - typedef typename value_alloc_type::const_pointer const_pointer; - typedef table_iterator<sparsetable<T, GROUP_SIZE, Alloc> > iterator; - typedef const_table_iterator<sparsetable<T, GROUP_SIZE, Alloc> > - const_iterator; - typedef table_element_adaptor<sparsetable<T, GROUP_SIZE, Alloc> > - element_adaptor; - typedef std::reverse_iterator<const_iterator> const_reverse_iterator; - typedef std::reverse_iterator<iterator> reverse_iterator; // from iterator.h - - // These are our special iterators, that go over non-empty buckets in a - // table. These aren't const only because you can change non-empty bcks. - typedef two_d_iterator< std::vector< sparsegroup<value_type, GROUP_SIZE, - value_alloc_type>, - vector_alloc> > - nonempty_iterator; - typedef const_two_d_iterator< std::vector< sparsegroup<value_type, - GROUP_SIZE, - value_alloc_type>, - vector_alloc> > - const_nonempty_iterator; - typedef std::reverse_iterator<nonempty_iterator> reverse_nonempty_iterator; - typedef std::reverse_iterator<const_nonempty_iterator> const_reverse_nonempty_iterator; - // Another special iterator: it frees memory as it iterates (used to resize) - typedef destructive_two_d_iterator< std::vector< sparsegroup<value_type, - GROUP_SIZE, - value_alloc_type>, - vector_alloc> > - destructive_iterator; - - // Iterator functions - iterator begin() { return iterator(this, 0); } - const_iterator begin() const { return const_iterator(this, 0); } - iterator end() { return iterator(this, size()); } - const_iterator end() const { return const_iterator(this, size()); } - reverse_iterator rbegin() { return reverse_iterator(end()); } - const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } - reverse_iterator rend() { return reverse_iterator(begin()); } - const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } - - // Versions for our special non-empty iterator - nonempty_iterator nonempty_begin() { - return nonempty_iterator(groups.begin(), groups.end(), groups.begin()); - } - const_nonempty_iterator nonempty_begin() const { - return const_nonempty_iterator(groups.begin(),groups.end(), groups.begin()); - } - nonempty_iterator nonempty_end() { - return nonempty_iterator(groups.begin(), groups.end(), groups.end()); - } - const_nonempty_iterator nonempty_end() const { - return const_nonempty_iterator(groups.begin(), groups.end(), groups.end()); - } - reverse_nonempty_iterator nonempty_rbegin() { - return reverse_nonempty_iterator(nonempty_end()); - } - const_reverse_nonempty_iterator nonempty_rbegin() const { - return const_reverse_nonempty_iterator(nonempty_end()); - } - reverse_nonempty_iterator nonempty_rend() { - return reverse_nonempty_iterator(nonempty_begin()); - } - const_reverse_nonempty_iterator nonempty_rend() const { - return const_reverse_nonempty_iterator(nonempty_begin()); - } - destructive_iterator destructive_begin() { - return destructive_iterator(groups.begin(), groups.end(), groups.begin()); - } - destructive_iterator destructive_end() { - return destructive_iterator(groups.begin(), groups.end(), groups.end()); - } - - typedef sparsegroup<value_type, GROUP_SIZE, allocator_type> group_type; - typedef std::vector<group_type, vector_alloc > group_vector_type; - - typedef typename group_vector_type::reference GroupsReference; - typedef typename group_vector_type::const_reference GroupsConstReference; - typedef typename group_vector_type::iterator GroupsIterator; - typedef typename group_vector_type::const_iterator GroupsConstIterator; - - // How to deal with the proper group - static size_type num_groups(size_type num) { // how many to hold num buckets - return num == 0 ? 0 : ((num-1) / GROUP_SIZE) + 1; - } - - u_int16_t pos_in_group(size_type i) const { - return static_cast<u_int16_t>(i % GROUP_SIZE); - } - size_type group_num(size_type i) const { - return i / GROUP_SIZE; - } - GroupsReference which_group(size_type i) { - return groups[group_num(i)]; - } - GroupsConstReference which_group(size_type i) const { - return groups[group_num(i)]; - } - - public: - // Constructors -- default, normal (when you specify size), and copy - explicit sparsetable(size_type sz = 0, Alloc alloc = Alloc()) - : groups(vector_alloc(alloc)), settings(alloc, sz) { - groups.resize(num_groups(sz), group_type(settings)); - } - // We can get away with using the default copy constructor, - // and default destructor, and hence the default operator=. Huzzah! - - // Many STL algorithms use swap instead of copy constructors - void swap(sparsetable& x) { - std::swap(groups, x.groups); // defined in stl_algobase.h - std::swap(settings.table_size, x.settings.table_size); - std::swap(settings.num_buckets, x.settings.num_buckets); - } - - // It's always nice to be able to clear a table without deallocating it - void clear() { - GroupsIterator group; - for ( group = groups.begin(); group != groups.end(); ++group ) { - group->clear(); - } - settings.num_buckets = 0; - } - - // ACCESSOR FUNCTIONS for the things we templatize on, basically - allocator_type get_allocator() const { - return allocator_type(settings); - } - - - // Functions that tell you about size. - // NOTE: empty() is non-intuitive! It does not tell you the number - // of not-empty buckets (use num_nonempty() for that). Instead - // it says whether you've allocated any buckets or not. - size_type size() const { return settings.table_size; } - size_type max_size() const { return settings.max_size(); } - bool empty() const { return settings.table_size == 0; } - // We also may want to know how many *used* buckets there are - size_type num_nonempty() const { return settings.num_buckets; } - - // OK, we'll let you resize one of these puppies - void resize(size_type new_size) { - groups.resize(num_groups(new_size), group_type(settings)); - if ( new_size < settings.table_size) { - // lower num_buckets, clear last group - if ( pos_in_group(new_size) > 0 ) // need to clear inside last group - groups.back().erase(groups.back().begin() + pos_in_group(new_size), - groups.back().end()); - settings.num_buckets = 0; // refigure # of used buckets - GroupsConstIterator group; - for ( group = groups.begin(); group != groups.end(); ++group ) - settings.num_buckets += group->num_nonempty(); - } - settings.table_size = new_size; - } - - - // We let you see if a bucket is non-empty without retrieving it - bool test(size_type i) const { - assert(i < settings.table_size); - return which_group(i).test(pos_in_group(i)); - } - bool test(iterator pos) const { - return which_group(pos.pos).test(pos_in_group(pos.pos)); - } - bool test(const_iterator pos) const { - return which_group(pos.pos).test(pos_in_group(pos.pos)); - } - - // We only return const_references because it's really hard to - // return something settable for empty buckets. Use set() instead. - const_reference get(size_type i) const { - assert(i < settings.table_size); - return which_group(i).get(pos_in_group(i)); - } - - // TODO(csilvers): make protected + friend - // This is used by sparse_hashtable to get an element from the table - // when we know it exists (because the caller has called test(i)). - const_reference unsafe_get(size_type i) const { - assert(i < settings.table_size); - assert(test(i)); - return which_group(i).unsafe_get(pos_in_group(i)); - } - - // TODO(csilvers): make protected + friend element_adaptor - reference mutating_get(size_type i) { // fills bucket i before getting - assert(i < settings.table_size); - typename group_type::size_type old_numbuckets = which_group(i).num_nonempty(); - reference retval = which_group(i).mutating_get(pos_in_group(i)); - settings.num_buckets += which_group(i).num_nonempty() - old_numbuckets; - return retval; - } - - // Syntactic sugar. As in sparsegroup, the non-const version is harder - const_reference operator[](size_type i) const { - return get(i); - } - - element_adaptor operator[](size_type i) { - return element_adaptor(this, i); - } - - // Needed for hashtables, gets as a nonempty_iterator. Crashes for empty bcks - const_nonempty_iterator get_iter(size_type i) const { - assert(test(i)); // how can a nonempty_iterator point to an empty bucket? - return const_nonempty_iterator( - groups.begin(), groups.end(), - groups.begin() + group_num(i), - (groups[group_num(i)].nonempty_begin() + - groups[group_num(i)].pos_to_offset(pos_in_group(i)))); - } - // For nonempty we can return a non-const version - nonempty_iterator get_iter(size_type i) { - assert(test(i)); // how can a nonempty_iterator point to an empty bucket? - return nonempty_iterator( - groups.begin(), groups.end(), - groups.begin() + group_num(i), - (groups[group_num(i)].nonempty_begin() + - groups[group_num(i)].pos_to_offset(pos_in_group(i)))); - } - - // And the reverse transformation. + group = settings.realloc_or_die(group, settings.num_buckets-1); + } + + // Shrink the array, without any special assumptions about value_type and + // allocator_type. + void erase_aux(size_type offset, base::false_type) { + // This is valid because 0 <= offset < num_buckets. Note the inequality. + pointer p = allocate_group(settings.num_buckets - 1); + std::uninitialized_copy(group, group + offset, p); + std::uninitialized_copy(group + offset + 1, group + settings.num_buckets, + p + offset); + free_group(); + group = p; + } + + public: + // This takes the specified elements out of the group. This is + // "undefining", rather than "clearing". + // TODO(austern): Make this exception safe: handle exceptions from + // value_type's copy constructor. + void erase(size_type i) { + if ( bmtest(i) ) { // trivial to erase empty bucket + size_type offset = pos_to_offset(bitmap,i); // where we'll find (or insert) + if ( settings.num_buckets == 1 ) { + free_group(); + group = NULL; + } else { + typedef base::integral_constant<bool, + (base::has_trivial_copy<value_type>::value && + base::has_trivial_destructor<value_type>::value && + base::is_same< + allocator_type, + libc_allocator_with_realloc<value_type> >::value)> + realloc_and_memmove_ok; // pretend mv(x,y) == "x.~T(); new(x) T(y)" + erase_aux(offset, realloc_and_memmove_ok()); + } + --settings.num_buckets; + bmclear(i); + } + } + + void erase(iterator pos) { + erase(pos.pos); + } + + void erase(iterator start_it, iterator end_it) { + // This could be more efficient, but to do so we'd need to make + // bmclear() clear a range of indices. Doesn't seem worth it. + for ( ; start_it != end_it; ++start_it ) + erase(start_it); + } + + + // I/O + // We support reading and writing groups to disk. We don't store + // the actual array contents (which we don't know how to store), + // just the bitmap and size. Meant to be used with table I/O. + + template <typename OUTPUT> bool write_metadata(OUTPUT *fp) const { + // we explicitly set to u_int16_t + assert(sizeof(settings.num_buckets) == 2); + if ( !sparsehash_internal::write_bigendian_number(fp, settings.num_buckets, + 2) ) + return false; + if ( !sparsehash_internal::write_data(fp, bitmap, sizeof(bitmap)) ) + return false; + return true; + } + + // Reading destroys the old group contents! Returns true if all was ok. + template <typename INPUT> bool read_metadata(INPUT *fp) { + clear(); + if ( !sparsehash_internal::read_bigendian_number(fp, &settings.num_buckets, + 2) ) + return false; + if ( !sparsehash_internal::read_data(fp, bitmap, sizeof(bitmap)) ) + return false; + // We'll allocate the space, but we won't fill it: it will be + // left as uninitialized raw memory. + group = allocate_group(settings.num_buckets); + return true; + } + + // Again, only meaningful if value_type is a POD. + template <typename INPUT> bool read_nopointer_data(INPUT *fp) { + for ( nonempty_iterator it = nonempty_begin(); + it != nonempty_end(); ++it ) { + if ( !sparsehash_internal::read_data(fp, &(*it), sizeof(*it)) ) + return false; + } + return true; + } + + // If your keys and values are simple enough, we can write them + // to disk for you. "simple enough" means POD and no pointers. + // However, we don't try to normalize endianness. + template <typename OUTPUT> bool write_nopointer_data(OUTPUT *fp) const { + for ( const_nonempty_iterator it = nonempty_begin(); + it != nonempty_end(); ++it ) { + if ( !sparsehash_internal::write_data(fp, &(*it), sizeof(*it)) ) + return false; + } + return true; + } + + + // Comparisons. We only need to define == and < -- we get + // != > <= >= via relops.h (which we happily included above). + // Note the comparisons are pretty arbitrary: we compare + // values of the first index that isn't equal (using default + // value for empty buckets). + bool operator==(const sparsegroup& x) const { + return ( settings.num_buckets == x.settings.num_buckets && + memcmp(bitmap, x.bitmap, sizeof(bitmap)) == 0 && + std::equal(begin(), end(), x.begin()) ); // from <algorithm> + } + + bool operator<(const sparsegroup& x) const { // also from <algorithm> + return std::lexicographical_compare(begin(), end(), x.begin(), x.end()); + } + bool operator!=(const sparsegroup& x) const { return !(*this == x); } + bool operator<=(const sparsegroup& x) const { return !(x < *this); } + bool operator>(const sparsegroup& x) const { return x < *this; } + bool operator>=(const sparsegroup& x) const { return !(*this < x); } + + private: + template <class A> + class alloc_impl : public A { + public: + typedef typename A::pointer pointer; + typedef typename A::size_type size_type; + + // Convert a normal allocator to one that has realloc_or_die() + alloc_impl(const A& a) : A(a) { } + + // realloc_or_die should only be used when using the default + // allocator (libc_allocator_with_realloc). + pointer realloc_or_die(pointer /*ptr*/, size_type /*n*/) { + fprintf(stderr, "realloc_or_die is only supported for " + "libc_allocator_with_realloc\n"); + exit(1); + return NULL; + } + }; + + // A template specialization of alloc_impl for + // libc_allocator_with_realloc that can handle realloc_or_die. + template <class A> + class alloc_impl<libc_allocator_with_realloc<A> > + : public libc_allocator_with_realloc<A> { + public: + typedef typename libc_allocator_with_realloc<A>::pointer pointer; + typedef typename libc_allocator_with_realloc<A>::size_type size_type; + + alloc_impl(const libc_allocator_with_realloc<A>& a) + : libc_allocator_with_realloc<A>(a) { } + + pointer realloc_or_die(pointer ptr, size_type n) { + pointer retval = this->reallocate(ptr, n); + if (retval == NULL) { + fprintf(stderr, "sparsehash: FATAL ERROR: failed to reallocate " + "%lu elements for ptr %p", static_cast<unsigned long>(n), ptr); + exit(1); + } + return retval; + } + }; + + // Package allocator with num_buckets to eliminate memory needed for the + // zero-size allocator. + // If new fields are added to this class, we should add them to + // operator= and swap. + class Settings : public alloc_impl<value_alloc_type> { + public: + Settings(const alloc_impl<value_alloc_type>& a, u_int16_t n = 0) + : alloc_impl<value_alloc_type>(a), num_buckets(n) { } + Settings(const Settings& s) + : alloc_impl<value_alloc_type>(s), num_buckets(s.num_buckets) { } + + u_int16_t num_buckets; // limits GROUP_SIZE to 64K + }; + + // The actual data + pointer group; // (small) array of T's + Settings settings; // allocator and num_buckets + unsigned char bitmap[(GROUP_SIZE-1)/8 + 1]; // fancy math is so we round up +}; + +// We need a global swap as well +template <class T, u_int16_t GROUP_SIZE, class Alloc> +inline void swap(sparsegroup<T,GROUP_SIZE,Alloc> &x, + sparsegroup<T,GROUP_SIZE,Alloc> &y) { + x.swap(y); +} + +// --------------------------------------------------------------------------- + + +template <class T, u_int16_t GROUP_SIZE = DEFAULT_SPARSEGROUP_SIZE, + class Alloc = libc_allocator_with_realloc<T> > +class sparsetable { + private: + typedef typename Alloc::template rebind<T>::other value_alloc_type; + typedef typename Alloc::template rebind< + sparsegroup<T, GROUP_SIZE, value_alloc_type> >::other vector_alloc; + + public: + // Basic types + typedef T value_type; // stolen from stl_vector.h + typedef Alloc allocator_type; + typedef typename value_alloc_type::size_type size_type; + typedef typename value_alloc_type::difference_type difference_type; + typedef typename value_alloc_type::reference reference; + typedef typename value_alloc_type::const_reference const_reference; + typedef typename value_alloc_type::pointer pointer; + typedef typename value_alloc_type::const_pointer const_pointer; + typedef table_iterator<sparsetable<T, GROUP_SIZE, Alloc> > iterator; + typedef const_table_iterator<sparsetable<T, GROUP_SIZE, Alloc> > + const_iterator; + typedef table_element_adaptor<sparsetable<T, GROUP_SIZE, Alloc> > + element_adaptor; + typedef std::reverse_iterator<const_iterator> const_reverse_iterator; + typedef std::reverse_iterator<iterator> reverse_iterator; // from iterator.h + + // These are our special iterators, that go over non-empty buckets in a + // table. These aren't const only because you can change non-empty bcks. + typedef two_d_iterator< std::vector< sparsegroup<value_type, GROUP_SIZE, + value_alloc_type>, + vector_alloc> > + nonempty_iterator; + typedef const_two_d_iterator< std::vector< sparsegroup<value_type, + GROUP_SIZE, + value_alloc_type>, + vector_alloc> > + const_nonempty_iterator; + typedef std::reverse_iterator<nonempty_iterator> reverse_nonempty_iterator; + typedef std::reverse_iterator<const_nonempty_iterator> const_reverse_nonempty_iterator; + // Another special iterator: it frees memory as it iterates (used to resize) + typedef destructive_two_d_iterator< std::vector< sparsegroup<value_type, + GROUP_SIZE, + value_alloc_type>, + vector_alloc> > + destructive_iterator; + + // Iterator functions + iterator begin() { return iterator(this, 0); } + const_iterator begin() const { return const_iterator(this, 0); } + iterator end() { return iterator(this, size()); } + const_iterator end() const { return const_iterator(this, size()); } + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } + + // Versions for our special non-empty iterator + nonempty_iterator nonempty_begin() { + return nonempty_iterator(groups.begin(), groups.end(), groups.begin()); + } + const_nonempty_iterator nonempty_begin() const { + return const_nonempty_iterator(groups.begin(),groups.end(), groups.begin()); + } + nonempty_iterator nonempty_end() { + return nonempty_iterator(groups.begin(), groups.end(), groups.end()); + } + const_nonempty_iterator nonempty_end() const { + return const_nonempty_iterator(groups.begin(), groups.end(), groups.end()); + } + reverse_nonempty_iterator nonempty_rbegin() { + return reverse_nonempty_iterator(nonempty_end()); + } + const_reverse_nonempty_iterator nonempty_rbegin() const { + return const_reverse_nonempty_iterator(nonempty_end()); + } + reverse_nonempty_iterator nonempty_rend() { + return reverse_nonempty_iterator(nonempty_begin()); + } + const_reverse_nonempty_iterator nonempty_rend() const { + return const_reverse_nonempty_iterator(nonempty_begin()); + } + destructive_iterator destructive_begin() { + return destructive_iterator(groups.begin(), groups.end(), groups.begin()); + } + destructive_iterator destructive_end() { + return destructive_iterator(groups.begin(), groups.end(), groups.end()); + } + + typedef sparsegroup<value_type, GROUP_SIZE, allocator_type> group_type; + typedef std::vector<group_type, vector_alloc > group_vector_type; + + typedef typename group_vector_type::reference GroupsReference; + typedef typename group_vector_type::const_reference GroupsConstReference; + typedef typename group_vector_type::iterator GroupsIterator; + typedef typename group_vector_type::const_iterator GroupsConstIterator; + + // How to deal with the proper group + static size_type num_groups(size_type num) { // how many to hold num buckets + return num == 0 ? 0 : ((num-1) / GROUP_SIZE) + 1; + } + + u_int16_t pos_in_group(size_type i) const { + return static_cast<u_int16_t>(i % GROUP_SIZE); + } + size_type group_num(size_type i) const { + return i / GROUP_SIZE; + } + GroupsReference which_group(size_type i) { + return groups[group_num(i)]; + } + GroupsConstReference which_group(size_type i) const { + return groups[group_num(i)]; + } + + public: + // Constructors -- default, normal (when you specify size), and copy + explicit sparsetable(size_type sz = 0, Alloc alloc = Alloc()) + : groups(vector_alloc(alloc)), settings(alloc, sz) { + groups.resize(num_groups(sz), group_type(settings)); + } + // We can get away with using the default copy constructor, + // and default destructor, and hence the default operator=. Huzzah! + + // Many STL algorithms use swap instead of copy constructors + void swap(sparsetable& x) { + std::swap(groups, x.groups); // defined in stl_algobase.h + std::swap(settings.table_size, x.settings.table_size); + std::swap(settings.num_buckets, x.settings.num_buckets); + } + + // It's always nice to be able to clear a table without deallocating it + void clear() { + GroupsIterator group; + for ( group = groups.begin(); group != groups.end(); ++group ) { + group->clear(); + } + settings.num_buckets = 0; + } + + // ACCESSOR FUNCTIONS for the things we templatize on, basically + allocator_type get_allocator() const { + return allocator_type(settings); + } + + + // Functions that tell you about size. + // NOTE: empty() is non-intuitive! It does not tell you the number + // of not-empty buckets (use num_nonempty() for that). Instead + // it says whether you've allocated any buckets or not. + size_type size() const { return settings.table_size; } + size_type max_size() const { return settings.max_size(); } + bool empty() const { return settings.table_size == 0; } + // We also may want to know how many *used* buckets there are + size_type num_nonempty() const { return settings.num_buckets; } + + // OK, we'll let you resize one of these puppies + void resize(size_type new_size) { + groups.resize(num_groups(new_size), group_type(settings)); + if ( new_size < settings.table_size) { + // lower num_buckets, clear last group + if ( pos_in_group(new_size) > 0 ) // need to clear inside last group + groups.back().erase(groups.back().begin() + pos_in_group(new_size), + groups.back().end()); + settings.num_buckets = 0; // refigure # of used buckets + GroupsConstIterator group; + for ( group = groups.begin(); group != groups.end(); ++group ) + settings.num_buckets += group->num_nonempty(); + } + settings.table_size = new_size; + } + + + // We let you see if a bucket is non-empty without retrieving it + bool test(size_type i) const { + assert(i < settings.table_size); + return which_group(i).test(pos_in_group(i)); + } + bool test(iterator pos) const { + return which_group(pos.pos).test(pos_in_group(pos.pos)); + } + bool test(const_iterator pos) const { + return which_group(pos.pos).test(pos_in_group(pos.pos)); + } + + // We only return const_references because it's really hard to + // return something settable for empty buckets. Use set() instead. + const_reference get(size_type i) const { + assert(i < settings.table_size); + return which_group(i).get(pos_in_group(i)); + } + + // TODO(csilvers): make protected + friend + // This is used by sparse_hashtable to get an element from the table + // when we know it exists (because the caller has called test(i)). + const_reference unsafe_get(size_type i) const { + assert(i < settings.table_size); + assert(test(i)); + return which_group(i).unsafe_get(pos_in_group(i)); + } + + // TODO(csilvers): make protected + friend element_adaptor + reference mutating_get(size_type i) { // fills bucket i before getting + assert(i < settings.table_size); + typename group_type::size_type old_numbuckets = which_group(i).num_nonempty(); + reference retval = which_group(i).mutating_get(pos_in_group(i)); + settings.num_buckets += which_group(i).num_nonempty() - old_numbuckets; + return retval; + } + + // Syntactic sugar. As in sparsegroup, the non-const version is harder + const_reference operator[](size_type i) const { + return get(i); + } + + element_adaptor operator[](size_type i) { + return element_adaptor(this, i); + } + + // Needed for hashtables, gets as a nonempty_iterator. Crashes for empty bcks + const_nonempty_iterator get_iter(size_type i) const { + assert(test(i)); // how can a nonempty_iterator point to an empty bucket? + return const_nonempty_iterator( + groups.begin(), groups.end(), + groups.begin() + group_num(i), + (groups[group_num(i)].nonempty_begin() + + groups[group_num(i)].pos_to_offset(pos_in_group(i)))); + } + // For nonempty we can return a non-const version + nonempty_iterator get_iter(size_type i) { + assert(test(i)); // how can a nonempty_iterator point to an empty bucket? + return nonempty_iterator( + groups.begin(), groups.end(), + groups.begin() + group_num(i), + (groups[group_num(i)].nonempty_begin() + + groups[group_num(i)].pos_to_offset(pos_in_group(i)))); + } + + // And the reverse transformation. size_type get_pos(const const_nonempty_iterator& it) const { - difference_type current_row = it.row_current - it.row_begin; - difference_type current_col = (it.col_current - - groups[current_row].nonempty_begin()); - return ((current_row * GROUP_SIZE) + - groups[current_row].offset_to_pos(current_col)); - } - - - // This returns a reference to the inserted item (which is a copy of val) - // The trick is to figure out whether we're replacing or inserting anew - reference set(size_type i, const_reference val) { - assert(i < settings.table_size); - typename group_type::size_type old_numbuckets = which_group(i).num_nonempty(); - reference retval = which_group(i).set(pos_in_group(i), val); - settings.num_buckets += which_group(i).num_nonempty() - old_numbuckets; - return retval; - } - - // This takes the specified elements out of the table. This is - // "undefining", rather than "clearing". - void erase(size_type i) { - assert(i < settings.table_size); - typename group_type::size_type old_numbuckets = which_group(i).num_nonempty(); - which_group(i).erase(pos_in_group(i)); - settings.num_buckets += which_group(i).num_nonempty() - old_numbuckets; - } - - void erase(iterator pos) { - erase(pos.pos); - } - - void erase(iterator start_it, iterator end_it) { - // This could be more efficient, but then we'd need to figure - // out if we spanned groups or not. Doesn't seem worth it. - for ( ; start_it != end_it; ++start_it ) - erase(start_it); - } - - - // We support reading and writing tables to disk. We don't store - // the actual array contents (which we don't know how to store), - // just the groups and sizes. Returns true if all went ok. - - private: - // Every time the disk format changes, this should probably change too - typedef unsigned long MagicNumberType; - static const MagicNumberType MAGIC_NUMBER = 0x24687531; - - // Old versions of this code write all data in 32 bits. We need to - // support these files as well as having support for 64-bit systems. - // So we use the following encoding scheme: for values < 2^32-1, we - // store in 4 bytes in big-endian order. For values > 2^32, we - // store 0xFFFFFFF followed by 8 bytes in big-endian order. This - // causes us to mis-read old-version code that stores exactly - // 0xFFFFFFF, but I don't think that is likely to have happened for - // these particular values. - template <typename OUTPUT, typename IntType> - static bool write_32_or_64(OUTPUT* fp, IntType value) { - if ( value < 0xFFFFFFFFULL ) { // fits in 4 bytes - if ( !sparsehash_internal::write_bigendian_number(fp, value, 4) ) - return false; - } else { - if ( !sparsehash_internal::write_bigendian_number(fp, 0xFFFFFFFFUL, 4) ) - return false; - if ( !sparsehash_internal::write_bigendian_number(fp, value, 8) ) - return false; - } - return true; - } - - template <typename INPUT, typename IntType> - static bool read_32_or_64(INPUT* fp, IntType *value) { // reads into value - MagicNumberType first4 = 0; // a convenient 32-bit unsigned type - if ( !sparsehash_internal::read_bigendian_number(fp, &first4, 4) ) - return false; - if ( first4 < 0xFFFFFFFFULL ) { - *value = first4; - } else { - if ( !sparsehash_internal::read_bigendian_number(fp, value, 8) ) - return false; - } - return true; - } - - public: - // read/write_metadata() and read_write/nopointer_data() are DEPRECATED. - // Use serialize() and unserialize(), below, for new code. - - template <typename OUTPUT> bool write_metadata(OUTPUT *fp) const { - if ( !write_32_or_64(fp, MAGIC_NUMBER) ) return false; - if ( !write_32_or_64(fp, settings.table_size) ) return false; - if ( !write_32_or_64(fp, settings.num_buckets) ) return false; - - GroupsConstIterator group; - for ( group = groups.begin(); group != groups.end(); ++group ) - if ( group->write_metadata(fp) == false ) return false; - return true; - } - - // Reading destroys the old table contents! Returns true if read ok. - template <typename INPUT> bool read_metadata(INPUT *fp) { - size_type magic_read = 0; - if ( !read_32_or_64(fp, &magic_read) ) return false; - if ( magic_read != MAGIC_NUMBER ) { - clear(); // just to be consistent - return false; - } - - if ( !read_32_or_64(fp, &settings.table_size) ) return false; - if ( !read_32_or_64(fp, &settings.num_buckets) ) return false; - - resize(settings.table_size); // so the vector's sized ok - GroupsIterator group; - for ( group = groups.begin(); group != groups.end(); ++group ) - if ( group->read_metadata(fp) == false ) return false; - return true; - } - - // This code is identical to that for SparseGroup - // If your keys and values are simple enough, we can write them - // to disk for you. "simple enough" means no pointers. - // However, we don't try to normalize endianness - bool write_nopointer_data(FILE *fp) const { - for ( const_nonempty_iterator it = nonempty_begin(); - it != nonempty_end(); ++it ) { - if ( !fwrite(&*it, sizeof(*it), 1, fp) ) return false; - } - return true; - } - - // When reading, we have to override the potential const-ness of *it - bool read_nopointer_data(FILE *fp) { - for ( nonempty_iterator it = nonempty_begin(); - it != nonempty_end(); ++it ) { - if ( !fread(reinterpret_cast<void*>(&(*it)), sizeof(*it), 1, fp) ) - return false; - } - return true; - } - - // INPUT and OUTPUT must be either a FILE, *or* a C++ stream - // (istream, ostream, etc) *or* a class providing - // Read(void*, size_t) and Write(const void*, size_t) - // (respectively), which writes a buffer into a stream - // (which the INPUT/OUTPUT instance presumably owns). - - typedef sparsehash_internal::pod_serializer<value_type> NopointerSerializer; - - // ValueSerializer: a functor. operator()(OUTPUT*, const value_type&) - template <typename ValueSerializer, typename OUTPUT> - bool serialize(ValueSerializer serializer, OUTPUT *fp) { - if ( !write_metadata(fp) ) - return false; - for ( const_nonempty_iterator it = nonempty_begin(); - it != nonempty_end(); ++it ) { - if ( !serializer(fp, *it) ) return false; - } - return true; - } - - // ValueSerializer: a functor. operator()(INPUT*, value_type*) - template <typename ValueSerializer, typename INPUT> - bool unserialize(ValueSerializer serializer, INPUT *fp) { - clear(); - if ( !read_metadata(fp) ) - return false; - for ( nonempty_iterator it = nonempty_begin(); - it != nonempty_end(); ++it ) { - if ( !serializer(fp, &*it) ) return false; - } - return true; - } - - // Comparisons. Note the comparisons are pretty arbitrary: we - // compare values of the first index that isn't equal (using default - // value for empty buckets). - bool operator==(const sparsetable& x) const { - return ( settings.table_size == x.settings.table_size && - settings.num_buckets == x.settings.num_buckets && - groups == x.groups ); - } - - bool operator<(const sparsetable& x) const { - return std::lexicographical_compare(begin(), end(), x.begin(), x.end()); - } - bool operator!=(const sparsetable& x) const { return !(*this == x); } - bool operator<=(const sparsetable& x) const { return !(x < *this); } - bool operator>(const sparsetable& x) const { return x < *this; } - bool operator>=(const sparsetable& x) const { return !(*this < x); } - - - private: - // Package allocator with table_size and num_buckets to eliminate memory - // needed for the zero-size allocator. - // If new fields are added to this class, we should add them to - // operator= and swap. - class Settings : public allocator_type { - public: - typedef typename allocator_type::size_type size_type; - - Settings(const allocator_type& a, size_type sz = 0, size_type n = 0) - : allocator_type(a), table_size(sz), num_buckets(n) { } - - Settings(const Settings& s) - : allocator_type(s), - table_size(s.table_size), num_buckets(s.num_buckets) { } - - size_type table_size; // how many buckets they want - size_type num_buckets; // number of non-empty buckets - }; - - // The actual data - group_vector_type groups; // our list of groups - Settings settings; // allocator, table size, buckets -}; - -// We need a global swap as well -template <class T, u_int16_t GROUP_SIZE, class Alloc> -inline void swap(sparsetable<T,GROUP_SIZE,Alloc> &x, - sparsetable<T,GROUP_SIZE,Alloc> &y) { - x.swap(y); -} - -_END_GOOGLE_NAMESPACE_ - -#endif // UTIL_GTL_SPARSETABLE_H_ + difference_type current_row = it.row_current - it.row_begin; + difference_type current_col = (it.col_current - + groups[current_row].nonempty_begin()); + return ((current_row * GROUP_SIZE) + + groups[current_row].offset_to_pos(current_col)); + } + + + // This returns a reference to the inserted item (which is a copy of val) + // The trick is to figure out whether we're replacing or inserting anew + reference set(size_type i, const_reference val) { + assert(i < settings.table_size); + typename group_type::size_type old_numbuckets = which_group(i).num_nonempty(); + reference retval = which_group(i).set(pos_in_group(i), val); + settings.num_buckets += which_group(i).num_nonempty() - old_numbuckets; + return retval; + } + + // This takes the specified elements out of the table. This is + // "undefining", rather than "clearing". + void erase(size_type i) { + assert(i < settings.table_size); + typename group_type::size_type old_numbuckets = which_group(i).num_nonempty(); + which_group(i).erase(pos_in_group(i)); + settings.num_buckets += which_group(i).num_nonempty() - old_numbuckets; + } + + void erase(iterator pos) { + erase(pos.pos); + } + + void erase(iterator start_it, iterator end_it) { + // This could be more efficient, but then we'd need to figure + // out if we spanned groups or not. Doesn't seem worth it. + for ( ; start_it != end_it; ++start_it ) + erase(start_it); + } + + + // We support reading and writing tables to disk. We don't store + // the actual array contents (which we don't know how to store), + // just the groups and sizes. Returns true if all went ok. + + private: + // Every time the disk format changes, this should probably change too + typedef unsigned long MagicNumberType; + static const MagicNumberType MAGIC_NUMBER = 0x24687531; + + // Old versions of this code write all data in 32 bits. We need to + // support these files as well as having support for 64-bit systems. + // So we use the following encoding scheme: for values < 2^32-1, we + // store in 4 bytes in big-endian order. For values > 2^32, we + // store 0xFFFFFFF followed by 8 bytes in big-endian order. This + // causes us to mis-read old-version code that stores exactly + // 0xFFFFFFF, but I don't think that is likely to have happened for + // these particular values. + template <typename OUTPUT, typename IntType> + static bool write_32_or_64(OUTPUT* fp, IntType value) { + if ( value < 0xFFFFFFFFULL ) { // fits in 4 bytes + if ( !sparsehash_internal::write_bigendian_number(fp, value, 4) ) + return false; + } else { + if ( !sparsehash_internal::write_bigendian_number(fp, 0xFFFFFFFFUL, 4) ) + return false; + if ( !sparsehash_internal::write_bigendian_number(fp, value, 8) ) + return false; + } + return true; + } + + template <typename INPUT, typename IntType> + static bool read_32_or_64(INPUT* fp, IntType *value) { // reads into value + MagicNumberType first4 = 0; // a convenient 32-bit unsigned type + if ( !sparsehash_internal::read_bigendian_number(fp, &first4, 4) ) + return false; + if ( first4 < 0xFFFFFFFFULL ) { + *value = first4; + } else { + if ( !sparsehash_internal::read_bigendian_number(fp, value, 8) ) + return false; + } + return true; + } + + public: + // read/write_metadata() and read_write/nopointer_data() are DEPRECATED. + // Use serialize() and unserialize(), below, for new code. + + template <typename OUTPUT> bool write_metadata(OUTPUT *fp) const { + if ( !write_32_or_64(fp, MAGIC_NUMBER) ) return false; + if ( !write_32_or_64(fp, settings.table_size) ) return false; + if ( !write_32_or_64(fp, settings.num_buckets) ) return false; + + GroupsConstIterator group; + for ( group = groups.begin(); group != groups.end(); ++group ) + if ( group->write_metadata(fp) == false ) return false; + return true; + } + + // Reading destroys the old table contents! Returns true if read ok. + template <typename INPUT> bool read_metadata(INPUT *fp) { + size_type magic_read = 0; + if ( !read_32_or_64(fp, &magic_read) ) return false; + if ( magic_read != MAGIC_NUMBER ) { + clear(); // just to be consistent + return false; + } + + if ( !read_32_or_64(fp, &settings.table_size) ) return false; + if ( !read_32_or_64(fp, &settings.num_buckets) ) return false; + + resize(settings.table_size); // so the vector's sized ok + GroupsIterator group; + for ( group = groups.begin(); group != groups.end(); ++group ) + if ( group->read_metadata(fp) == false ) return false; + return true; + } + + // This code is identical to that for SparseGroup + // If your keys and values are simple enough, we can write them + // to disk for you. "simple enough" means no pointers. + // However, we don't try to normalize endianness + bool write_nopointer_data(FILE *fp) const { + for ( const_nonempty_iterator it = nonempty_begin(); + it != nonempty_end(); ++it ) { + if ( !fwrite(&*it, sizeof(*it), 1, fp) ) return false; + } + return true; + } + + // When reading, we have to override the potential const-ness of *it + bool read_nopointer_data(FILE *fp) { + for ( nonempty_iterator it = nonempty_begin(); + it != nonempty_end(); ++it ) { + if ( !fread(reinterpret_cast<void*>(&(*it)), sizeof(*it), 1, fp) ) + return false; + } + return true; + } + + // INPUT and OUTPUT must be either a FILE, *or* a C++ stream + // (istream, ostream, etc) *or* a class providing + // Read(void*, size_t) and Write(const void*, size_t) + // (respectively), which writes a buffer into a stream + // (which the INPUT/OUTPUT instance presumably owns). + + typedef sparsehash_internal::pod_serializer<value_type> NopointerSerializer; + + // ValueSerializer: a functor. operator()(OUTPUT*, const value_type&) + template <typename ValueSerializer, typename OUTPUT> + bool serialize(ValueSerializer serializer, OUTPUT *fp) { + if ( !write_metadata(fp) ) + return false; + for ( const_nonempty_iterator it = nonempty_begin(); + it != nonempty_end(); ++it ) { + if ( !serializer(fp, *it) ) return false; + } + return true; + } + + // ValueSerializer: a functor. operator()(INPUT*, value_type*) + template <typename ValueSerializer, typename INPUT> + bool unserialize(ValueSerializer serializer, INPUT *fp) { + clear(); + if ( !read_metadata(fp) ) + return false; + for ( nonempty_iterator it = nonempty_begin(); + it != nonempty_end(); ++it ) { + if ( !serializer(fp, &*it) ) return false; + } + return true; + } + + // Comparisons. Note the comparisons are pretty arbitrary: we + // compare values of the first index that isn't equal (using default + // value for empty buckets). + bool operator==(const sparsetable& x) const { + return ( settings.table_size == x.settings.table_size && + settings.num_buckets == x.settings.num_buckets && + groups == x.groups ); + } + + bool operator<(const sparsetable& x) const { + return std::lexicographical_compare(begin(), end(), x.begin(), x.end()); + } + bool operator!=(const sparsetable& x) const { return !(*this == x); } + bool operator<=(const sparsetable& x) const { return !(x < *this); } + bool operator>(const sparsetable& x) const { return x < *this; } + bool operator>=(const sparsetable& x) const { return !(*this < x); } + + + private: + // Package allocator with table_size and num_buckets to eliminate memory + // needed for the zero-size allocator. + // If new fields are added to this class, we should add them to + // operator= and swap. + class Settings : public allocator_type { + public: + typedef typename allocator_type::size_type size_type; + + Settings(const allocator_type& a, size_type sz = 0, size_type n = 0) + : allocator_type(a), table_size(sz), num_buckets(n) { } + + Settings(const Settings& s) + : allocator_type(s), + table_size(s.table_size), num_buckets(s.num_buckets) { } + + size_type table_size; // how many buckets they want + size_type num_buckets; // number of non-empty buckets + }; + + // The actual data + group_vector_type groups; // our list of groups + Settings settings; // allocator, table size, buckets +}; + +// We need a global swap as well +template <class T, u_int16_t GROUP_SIZE, class Alloc> +inline void swap(sparsetable<T,GROUP_SIZE,Alloc> &x, + sparsetable<T,GROUP_SIZE,Alloc> &y) { + x.swap(y); +} + +_END_GOOGLE_NAMESPACE_ + +#endif // UTIL_GTL_SPARSETABLE_H_ diff --git a/contrib/libs/sparsehash/src/sparsehash/template_util.h b/contrib/libs/sparsehash/src/sparsehash/template_util.h index 182cef24a1..6fec3d0924 100644 --- a/contrib/libs/sparsehash/src/sparsehash/template_util.h +++ b/contrib/libs/sparsehash/src/sparsehash/template_util.h @@ -1,134 +1,134 @@ -// Copyright 2005 Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// ---- -// -// Template metaprogramming utility functions. -// -// This code is compiled directly on many platforms, including client -// platforms like Windows, Mac, and embedded systems. Before making -// any changes here, make sure that you're not breaking any platforms. -// -// -// The names choosen here reflect those used in tr1 and the boost::mpl -// library, there are similar operations used in the Loki library as -// well. I prefer the boost names for 2 reasons: -// 1. I think that portions of the Boost libraries are more likely to -// be included in the c++ standard. -// 2. It is not impossible that some of the boost libraries will be -// included in our own build in the future. -// Both of these outcomes means that we may be able to directly replace -// some of these with boost equivalents. -// -#ifndef BASE_TEMPLATE_UTIL_H_ -#define BASE_TEMPLATE_UTIL_H_ - -#include <sparsehash/internal/sparseconfig.h> -_START_GOOGLE_NAMESPACE_ - -// Types small_ and big_ are guaranteed such that sizeof(small_) < -// sizeof(big_) -typedef char small_; - -struct big_ { - char dummy[2]; -}; - -// Identity metafunction. -template <class T> -struct identity_ { - typedef T type; -}; - -// integral_constant, defined in tr1, is a wrapper for an integer -// value. We don't really need this generality; we could get away -// with hardcoding the integer type to bool. We use the fully -// general integer_constant for compatibility with tr1. - -template<class T, T v> -struct integral_constant { - static const T value = v; - typedef T value_type; - typedef integral_constant<T, v> type; -}; - -template <class T, T v> const T integral_constant<T, v>::value; - - -// Abbreviations: true_type and false_type are structs that represent boolean -// true and false values. Also define the boost::mpl versions of those names, -// true_ and false_. -typedef integral_constant<bool, true> true_type; -typedef integral_constant<bool, false> false_type; -typedef true_type true_; -typedef false_type false_; - -// if_ is a templatized conditional statement. -// if_<cond, A, B> is a compile time evaluation of cond. -// if_<>::type contains A if cond is true, B otherwise. -template<bool cond, typename A, typename B> -struct if_{ - typedef A type; -}; - -template<typename A, typename B> -struct if_<false, A, B> { - typedef B type; -}; - - -// type_equals_ is a template type comparator, similar to Loki IsSameType. -// type_equals_<A, B>::value is true iff "A" is the same type as "B". -// -// New code should prefer base::is_same, defined in base/type_traits.h. -// It is functionally identical, but is_same is the standard spelling. -template<typename A, typename B> -struct type_equals_ : public false_ { -}; - -template<typename A> -struct type_equals_<A, A> : public true_ { -}; - -// and_ is a template && operator. -// and_<A, B>::value evaluates "A::value && B::value". -template<typename A, typename B> -struct and_ : public integral_constant<bool, (A::value && B::value)> { -}; - -// or_ is a template || operator. -// or_<A, B>::value evaluates "A::value || B::value". -template<typename A, typename B> -struct or_ : public integral_constant<bool, (A::value || B::value)> { -}; - - -_END_GOOGLE_NAMESPACE_ - -#endif // BASE_TEMPLATE_UTIL_H_ +// Copyright 2005 Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ---- +// +// Template metaprogramming utility functions. +// +// This code is compiled directly on many platforms, including client +// platforms like Windows, Mac, and embedded systems. Before making +// any changes here, make sure that you're not breaking any platforms. +// +// +// The names choosen here reflect those used in tr1 and the boost::mpl +// library, there are similar operations used in the Loki library as +// well. I prefer the boost names for 2 reasons: +// 1. I think that portions of the Boost libraries are more likely to +// be included in the c++ standard. +// 2. It is not impossible that some of the boost libraries will be +// included in our own build in the future. +// Both of these outcomes means that we may be able to directly replace +// some of these with boost equivalents. +// +#ifndef BASE_TEMPLATE_UTIL_H_ +#define BASE_TEMPLATE_UTIL_H_ + +#include <sparsehash/internal/sparseconfig.h> +_START_GOOGLE_NAMESPACE_ + +// Types small_ and big_ are guaranteed such that sizeof(small_) < +// sizeof(big_) +typedef char small_; + +struct big_ { + char dummy[2]; +}; + +// Identity metafunction. +template <class T> +struct identity_ { + typedef T type; +}; + +// integral_constant, defined in tr1, is a wrapper for an integer +// value. We don't really need this generality; we could get away +// with hardcoding the integer type to bool. We use the fully +// general integer_constant for compatibility with tr1. + +template<class T, T v> +struct integral_constant { + static const T value = v; + typedef T value_type; + typedef integral_constant<T, v> type; +}; + +template <class T, T v> const T integral_constant<T, v>::value; + + +// Abbreviations: true_type and false_type are structs that represent boolean +// true and false values. Also define the boost::mpl versions of those names, +// true_ and false_. +typedef integral_constant<bool, true> true_type; +typedef integral_constant<bool, false> false_type; +typedef true_type true_; +typedef false_type false_; + +// if_ is a templatized conditional statement. +// if_<cond, A, B> is a compile time evaluation of cond. +// if_<>::type contains A if cond is true, B otherwise. +template<bool cond, typename A, typename B> +struct if_{ + typedef A type; +}; + +template<typename A, typename B> +struct if_<false, A, B> { + typedef B type; +}; + + +// type_equals_ is a template type comparator, similar to Loki IsSameType. +// type_equals_<A, B>::value is true iff "A" is the same type as "B". +// +// New code should prefer base::is_same, defined in base/type_traits.h. +// It is functionally identical, but is_same is the standard spelling. +template<typename A, typename B> +struct type_equals_ : public false_ { +}; + +template<typename A> +struct type_equals_<A, A> : public true_ { +}; + +// and_ is a template && operator. +// and_<A, B>::value evaluates "A::value && B::value". +template<typename A, typename B> +struct and_ : public integral_constant<bool, (A::value && B::value)> { +}; + +// or_ is a template || operator. +// or_<A, B>::value evaluates "A::value || B::value". +template<typename A, typename B> +struct or_ : public integral_constant<bool, (A::value || B::value)> { +}; + + +_END_GOOGLE_NAMESPACE_ + +#endif // BASE_TEMPLATE_UTIL_H_ diff --git a/contrib/libs/sparsehash/src/sparsehash/type_traits.h b/contrib/libs/sparsehash/src/sparsehash/type_traits.h index 478d9068a7..f909cf9a37 100644 --- a/contrib/libs/sparsehash/src/sparsehash/type_traits.h +++ b/contrib/libs/sparsehash/src/sparsehash/type_traits.h @@ -1,342 +1,342 @@ -// Copyright (c) 2006, Google Inc. -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// ---- -// -// This code is compiled directly on many platforms, including client -// platforms like Windows, Mac, and embedded systems. Before making -// any changes here, make sure that you're not breaking any platforms. -// -// Define a small subset of tr1 type traits. The traits we define are: -// is_integral -// is_floating_point -// is_pointer -// is_enum -// is_reference -// is_pod -// has_trivial_constructor -// has_trivial_copy -// has_trivial_assign -// has_trivial_destructor -// remove_const -// remove_volatile -// remove_cv -// remove_reference -// add_reference -// remove_pointer -// is_same -// is_convertible -// We can add more type traits as required. - -#ifndef BASE_TYPE_TRAITS_H_ -#define BASE_TYPE_TRAITS_H_ - -#include <sparsehash/internal/sparseconfig.h> -#include <utility> // For pair - -#include <sparsehash/template_util.h> // For true_type and false_type - -_START_GOOGLE_NAMESPACE_ - -template <class T> struct is_integral; -template <class T> struct is_floating_point; -template <class T> struct is_pointer; -// MSVC can't compile this correctly, and neither can gcc 3.3.5 (at least) -#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) -// is_enum uses is_convertible, which is not available on MSVC. -template <class T> struct is_enum; -#endif -template <class T> struct is_reference; -template <class T> struct is_pod; -template <class T> struct has_trivial_constructor; -template <class T> struct has_trivial_copy; -template <class T> struct has_trivial_assign; -template <class T> struct has_trivial_destructor; -template <class T> struct remove_const; -template <class T> struct remove_volatile; -template <class T> struct remove_cv; -template <class T> struct remove_reference; -template <class T> struct add_reference; -template <class T> struct remove_pointer; -template <class T, class U> struct is_same; -#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) -template <class From, class To> struct is_convertible; -#endif - -// is_integral is false except for the built-in integer types. A -// cv-qualified type is integral if and only if the underlying type is. -template <class T> struct is_integral : false_type { }; -template<> struct is_integral<bool> : true_type { }; -template<> struct is_integral<char> : true_type { }; -template<> struct is_integral<unsigned char> : true_type { }; -template<> struct is_integral<signed char> : true_type { }; -#if defined(_MSC_VER) -// wchar_t is not by default a distinct type from unsigned short in -// Microsoft C. -// See http://msdn2.microsoft.com/en-us/library/dh8che7s(VS.80).aspx -template<> struct is_integral<__wchar_t> : true_type { }; -#else -template<> struct is_integral<wchar_t> : true_type { }; -#endif -template<> struct is_integral<short> : true_type { }; -template<> struct is_integral<unsigned short> : true_type { }; -template<> struct is_integral<int> : true_type { }; -template<> struct is_integral<unsigned int> : true_type { }; -template<> struct is_integral<long> : true_type { }; -template<> struct is_integral<unsigned long> : true_type { }; -#ifdef HAVE_LONG_LONG -template<> struct is_integral<long long> : true_type { }; -template<> struct is_integral<unsigned long long> : true_type { }; -#endif -template <class T> struct is_integral<const T> : is_integral<T> { }; -template <class T> struct is_integral<volatile T> : is_integral<T> { }; -template <class T> struct is_integral<const volatile T> : is_integral<T> { }; - -// is_floating_point is false except for the built-in floating-point types. -// A cv-qualified type is integral if and only if the underlying type is. -template <class T> struct is_floating_point : false_type { }; -template<> struct is_floating_point<float> : true_type { }; -template<> struct is_floating_point<double> : true_type { }; -template<> struct is_floating_point<long double> : true_type { }; -template <class T> struct is_floating_point<const T> - : is_floating_point<T> { }; -template <class T> struct is_floating_point<volatile T> - : is_floating_point<T> { }; -template <class T> struct is_floating_point<const volatile T> - : is_floating_point<T> { }; - -// is_pointer is false except for pointer types. A cv-qualified type (e.g. -// "int* const", as opposed to "int const*") is cv-qualified if and only if -// the underlying type is. -template <class T> struct is_pointer : false_type { }; -template <class T> struct is_pointer<T*> : true_type { }; -template <class T> struct is_pointer<const T> : is_pointer<T> { }; -template <class T> struct is_pointer<volatile T> : is_pointer<T> { }; -template <class T> struct is_pointer<const volatile T> : is_pointer<T> { }; - -#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) - -namespace internal { - -template <class T> struct is_class_or_union { - template <class U> static small_ tester(void (U::*)()); - template <class U> static big_ tester(...); - static const bool value = sizeof(tester<T>(0)) == sizeof(small_); -}; - -// is_convertible chokes if the first argument is an array. That's why -// we use add_reference here. -template <bool NotUnum, class T> struct is_enum_impl - : is_convertible<typename add_reference<T>::type, int> { }; - -template <class T> struct is_enum_impl<true, T> : false_type { }; - -} // namespace internal - -// Specified by TR1 [4.5.1] primary type categories. - -// Implementation note: -// -// Each type is either void, integral, floating point, array, pointer, -// reference, member object pointer, member function pointer, enum, -// union or class. Out of these, only integral, floating point, reference, -// class and enum types are potentially convertible to int. Therefore, -// if a type is not a reference, integral, floating point or class and -// is convertible to int, it's a enum. Adding cv-qualification to a type -// does not change whether it's an enum. -// -// Is-convertible-to-int check is done only if all other checks pass, -// because it can't be used with some types (e.g. void or classes with -// inaccessible conversion operators). -template <class T> struct is_enum - : internal::is_enum_impl< - is_same<T, void>::value || - is_integral<T>::value || - is_floating_point<T>::value || - is_reference<T>::value || - internal::is_class_or_union<T>::value, - T> { }; - -template <class T> struct is_enum<const T> : is_enum<T> { }; -template <class T> struct is_enum<volatile T> : is_enum<T> { }; -template <class T> struct is_enum<const volatile T> : is_enum<T> { }; - -#endif - -// is_reference is false except for reference types. -template<typename T> struct is_reference : false_type {}; -template<typename T> struct is_reference<T&> : true_type {}; - - -// We can't get is_pod right without compiler help, so fail conservatively. -// We will assume it's false except for arithmetic types, enumerations, -// pointers and cv-qualified versions thereof. Note that std::pair<T,U> -// is not a POD even if T and U are PODs. -template <class T> struct is_pod - : integral_constant<bool, (is_integral<T>::value || - is_floating_point<T>::value || -#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) - // is_enum is not available on MSVC. - is_enum<T>::value || -#endif - is_pointer<T>::value)> { }; -template <class T> struct is_pod<const T> : is_pod<T> { }; -template <class T> struct is_pod<volatile T> : is_pod<T> { }; -template <class T> struct is_pod<const volatile T> : is_pod<T> { }; - - -// We can't get has_trivial_constructor right without compiler help, so -// fail conservatively. We will assume it's false except for: (1) types -// for which is_pod is true. (2) std::pair of types with trivial -// constructors. (3) array of a type with a trivial constructor. -// (4) const versions thereof. -template <class T> struct has_trivial_constructor : is_pod<T> { }; -template <class T, class U> struct has_trivial_constructor<std::pair<T, U> > - : integral_constant<bool, - (has_trivial_constructor<T>::value && - has_trivial_constructor<U>::value)> { }; -template <class A, int N> struct has_trivial_constructor<A[N]> - : has_trivial_constructor<A> { }; -template <class T> struct has_trivial_constructor<const T> - : has_trivial_constructor<T> { }; - -// We can't get has_trivial_copy right without compiler help, so fail -// conservatively. We will assume it's false except for: (1) types -// for which is_pod is true. (2) std::pair of types with trivial copy -// constructors. (3) array of a type with a trivial copy constructor. -// (4) const versions thereof. -template <class T> struct has_trivial_copy : is_pod<T> { }; -template <class T, class U> struct has_trivial_copy<std::pair<T, U> > - : integral_constant<bool, - (has_trivial_copy<T>::value && - has_trivial_copy<U>::value)> { }; -template <class A, int N> struct has_trivial_copy<A[N]> - : has_trivial_copy<A> { }; -template <class T> struct has_trivial_copy<const T> : has_trivial_copy<T> { }; - -// We can't get has_trivial_assign right without compiler help, so fail -// conservatively. We will assume it's false except for: (1) types -// for which is_pod is true. (2) std::pair of types with trivial copy -// constructors. (3) array of a type with a trivial assign constructor. -template <class T> struct has_trivial_assign : is_pod<T> { }; -template <class T, class U> struct has_trivial_assign<std::pair<T, U> > - : integral_constant<bool, - (has_trivial_assign<T>::value && - has_trivial_assign<U>::value)> { }; -template <class A, int N> struct has_trivial_assign<A[N]> - : has_trivial_assign<A> { }; - -// We can't get has_trivial_destructor right without compiler help, so -// fail conservatively. We will assume it's false except for: (1) types -// for which is_pod is true. (2) std::pair of types with trivial -// destructors. (3) array of a type with a trivial destructor. -// (4) const versions thereof. -template <class T> struct has_trivial_destructor : is_pod<T> { }; -template <class T, class U> struct has_trivial_destructor<std::pair<T, U> > - : integral_constant<bool, - (has_trivial_destructor<T>::value && - has_trivial_destructor<U>::value)> { }; -template <class A, int N> struct has_trivial_destructor<A[N]> - : has_trivial_destructor<A> { }; -template <class T> struct has_trivial_destructor<const T> - : has_trivial_destructor<T> { }; - -// Specified by TR1 [4.7.1] -template<typename T> struct remove_const { typedef T type; }; -template<typename T> struct remove_const<T const> { typedef T type; }; -template<typename T> struct remove_volatile { typedef T type; }; -template<typename T> struct remove_volatile<T volatile> { typedef T type; }; -template<typename T> struct remove_cv { - typedef typename remove_const<typename remove_volatile<T>::type>::type type; -}; - - -// Specified by TR1 [4.7.2] Reference modifications. -template<typename T> struct remove_reference { typedef T type; }; -template<typename T> struct remove_reference<T&> { typedef T type; }; - -template <typename T> struct add_reference { typedef T& type; }; -template <typename T> struct add_reference<T&> { typedef T& type; }; - -// Specified by TR1 [4.7.4] Pointer modifications. -template<typename T> struct remove_pointer { typedef T type; }; -template<typename T> struct remove_pointer<T*> { typedef T type; }; -template<typename T> struct remove_pointer<T* const> { typedef T type; }; -template<typename T> struct remove_pointer<T* volatile> { typedef T type; }; -template<typename T> struct remove_pointer<T* const volatile> { - typedef T type; }; - -// Specified by TR1 [4.6] Relationships between types -template<typename T, typename U> struct is_same : public false_type { }; -template<typename T> struct is_same<T, T> : public true_type { }; - -// Specified by TR1 [4.6] Relationships between types -#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) -namespace internal { - -// This class is an implementation detail for is_convertible, and you -// don't need to know how it works to use is_convertible. For those -// who care: we declare two different functions, one whose argument is -// of type To and one with a variadic argument list. We give them -// return types of different size, so we can use sizeof to trick the -// compiler into telling us which function it would have chosen if we -// had called it with an argument of type From. See Alexandrescu's -// _Modern C++ Design_ for more details on this sort of trick. - -template <typename From, typename To> -struct ConvertHelper { - static small_ Test(To); - static big_ Test(...); - static From Create(); -}; -} // namespace internal - -// Inherits from true_type if From is convertible to To, false_type otherwise. -template <typename From, typename To> -struct is_convertible - : integral_constant<bool, - sizeof(internal::ConvertHelper<From, To>::Test( - internal::ConvertHelper<From, To>::Create())) - == sizeof(small_)> { -}; -#endif - -_END_GOOGLE_NAMESPACE_ - -// Right now these macros are no-ops, and mostly just document the fact -// these types are PODs, for human use. They may be made more contentful -// later. The typedef is just to make it legal to put a semicolon after -// these macros. -#define DECLARE_POD(TypeName) typedef int Dummy_Type_For_DECLARE_POD -#define DECLARE_NESTED_POD(TypeName) DECLARE_POD(TypeName) -#define PROPAGATE_POD_FROM_TEMPLATE_ARGUMENT(TemplateName) \ - typedef int Dummy_Type_For_PROPAGATE_POD_FROM_TEMPLATE_ARGUMENT -#define ENFORCE_POD(TypeName) typedef int Dummy_Type_For_ENFORCE_POD - -#endif // BASE_TYPE_TRAITS_H_ +// Copyright (c) 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ---- +// +// This code is compiled directly on many platforms, including client +// platforms like Windows, Mac, and embedded systems. Before making +// any changes here, make sure that you're not breaking any platforms. +// +// Define a small subset of tr1 type traits. The traits we define are: +// is_integral +// is_floating_point +// is_pointer +// is_enum +// is_reference +// is_pod +// has_trivial_constructor +// has_trivial_copy +// has_trivial_assign +// has_trivial_destructor +// remove_const +// remove_volatile +// remove_cv +// remove_reference +// add_reference +// remove_pointer +// is_same +// is_convertible +// We can add more type traits as required. + +#ifndef BASE_TYPE_TRAITS_H_ +#define BASE_TYPE_TRAITS_H_ + +#include <sparsehash/internal/sparseconfig.h> +#include <utility> // For pair + +#include <sparsehash/template_util.h> // For true_type and false_type + +_START_GOOGLE_NAMESPACE_ + +template <class T> struct is_integral; +template <class T> struct is_floating_point; +template <class T> struct is_pointer; +// MSVC can't compile this correctly, and neither can gcc 3.3.5 (at least) +#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) +// is_enum uses is_convertible, which is not available on MSVC. +template <class T> struct is_enum; +#endif +template <class T> struct is_reference; +template <class T> struct is_pod; +template <class T> struct has_trivial_constructor; +template <class T> struct has_trivial_copy; +template <class T> struct has_trivial_assign; +template <class T> struct has_trivial_destructor; +template <class T> struct remove_const; +template <class T> struct remove_volatile; +template <class T> struct remove_cv; +template <class T> struct remove_reference; +template <class T> struct add_reference; +template <class T> struct remove_pointer; +template <class T, class U> struct is_same; +#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) +template <class From, class To> struct is_convertible; +#endif + +// is_integral is false except for the built-in integer types. A +// cv-qualified type is integral if and only if the underlying type is. +template <class T> struct is_integral : false_type { }; +template<> struct is_integral<bool> : true_type { }; +template<> struct is_integral<char> : true_type { }; +template<> struct is_integral<unsigned char> : true_type { }; +template<> struct is_integral<signed char> : true_type { }; +#if defined(_MSC_VER) +// wchar_t is not by default a distinct type from unsigned short in +// Microsoft C. +// See http://msdn2.microsoft.com/en-us/library/dh8che7s(VS.80).aspx +template<> struct is_integral<__wchar_t> : true_type { }; +#else +template<> struct is_integral<wchar_t> : true_type { }; +#endif +template<> struct is_integral<short> : true_type { }; +template<> struct is_integral<unsigned short> : true_type { }; +template<> struct is_integral<int> : true_type { }; +template<> struct is_integral<unsigned int> : true_type { }; +template<> struct is_integral<long> : true_type { }; +template<> struct is_integral<unsigned long> : true_type { }; +#ifdef HAVE_LONG_LONG +template<> struct is_integral<long long> : true_type { }; +template<> struct is_integral<unsigned long long> : true_type { }; +#endif +template <class T> struct is_integral<const T> : is_integral<T> { }; +template <class T> struct is_integral<volatile T> : is_integral<T> { }; +template <class T> struct is_integral<const volatile T> : is_integral<T> { }; + +// is_floating_point is false except for the built-in floating-point types. +// A cv-qualified type is integral if and only if the underlying type is. +template <class T> struct is_floating_point : false_type { }; +template<> struct is_floating_point<float> : true_type { }; +template<> struct is_floating_point<double> : true_type { }; +template<> struct is_floating_point<long double> : true_type { }; +template <class T> struct is_floating_point<const T> + : is_floating_point<T> { }; +template <class T> struct is_floating_point<volatile T> + : is_floating_point<T> { }; +template <class T> struct is_floating_point<const volatile T> + : is_floating_point<T> { }; + +// is_pointer is false except for pointer types. A cv-qualified type (e.g. +// "int* const", as opposed to "int const*") is cv-qualified if and only if +// the underlying type is. +template <class T> struct is_pointer : false_type { }; +template <class T> struct is_pointer<T*> : true_type { }; +template <class T> struct is_pointer<const T> : is_pointer<T> { }; +template <class T> struct is_pointer<volatile T> : is_pointer<T> { }; +template <class T> struct is_pointer<const volatile T> : is_pointer<T> { }; + +#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) + +namespace internal { + +template <class T> struct is_class_or_union { + template <class U> static small_ tester(void (U::*)()); + template <class U> static big_ tester(...); + static const bool value = sizeof(tester<T>(0)) == sizeof(small_); +}; + +// is_convertible chokes if the first argument is an array. That's why +// we use add_reference here. +template <bool NotUnum, class T> struct is_enum_impl + : is_convertible<typename add_reference<T>::type, int> { }; + +template <class T> struct is_enum_impl<true, T> : false_type { }; + +} // namespace internal + +// Specified by TR1 [4.5.1] primary type categories. + +// Implementation note: +// +// Each type is either void, integral, floating point, array, pointer, +// reference, member object pointer, member function pointer, enum, +// union or class. Out of these, only integral, floating point, reference, +// class and enum types are potentially convertible to int. Therefore, +// if a type is not a reference, integral, floating point or class and +// is convertible to int, it's a enum. Adding cv-qualification to a type +// does not change whether it's an enum. +// +// Is-convertible-to-int check is done only if all other checks pass, +// because it can't be used with some types (e.g. void or classes with +// inaccessible conversion operators). +template <class T> struct is_enum + : internal::is_enum_impl< + is_same<T, void>::value || + is_integral<T>::value || + is_floating_point<T>::value || + is_reference<T>::value || + internal::is_class_or_union<T>::value, + T> { }; + +template <class T> struct is_enum<const T> : is_enum<T> { }; +template <class T> struct is_enum<volatile T> : is_enum<T> { }; +template <class T> struct is_enum<const volatile T> : is_enum<T> { }; + +#endif + +// is_reference is false except for reference types. +template<typename T> struct is_reference : false_type {}; +template<typename T> struct is_reference<T&> : true_type {}; + + +// We can't get is_pod right without compiler help, so fail conservatively. +// We will assume it's false except for arithmetic types, enumerations, +// pointers and cv-qualified versions thereof. Note that std::pair<T,U> +// is not a POD even if T and U are PODs. +template <class T> struct is_pod + : integral_constant<bool, (is_integral<T>::value || + is_floating_point<T>::value || +#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) + // is_enum is not available on MSVC. + is_enum<T>::value || +#endif + is_pointer<T>::value)> { }; +template <class T> struct is_pod<const T> : is_pod<T> { }; +template <class T> struct is_pod<volatile T> : is_pod<T> { }; +template <class T> struct is_pod<const volatile T> : is_pod<T> { }; + + +// We can't get has_trivial_constructor right without compiler help, so +// fail conservatively. We will assume it's false except for: (1) types +// for which is_pod is true. (2) std::pair of types with trivial +// constructors. (3) array of a type with a trivial constructor. +// (4) const versions thereof. +template <class T> struct has_trivial_constructor : is_pod<T> { }; +template <class T, class U> struct has_trivial_constructor<std::pair<T, U> > + : integral_constant<bool, + (has_trivial_constructor<T>::value && + has_trivial_constructor<U>::value)> { }; +template <class A, int N> struct has_trivial_constructor<A[N]> + : has_trivial_constructor<A> { }; +template <class T> struct has_trivial_constructor<const T> + : has_trivial_constructor<T> { }; + +// We can't get has_trivial_copy right without compiler help, so fail +// conservatively. We will assume it's false except for: (1) types +// for which is_pod is true. (2) std::pair of types with trivial copy +// constructors. (3) array of a type with a trivial copy constructor. +// (4) const versions thereof. +template <class T> struct has_trivial_copy : is_pod<T> { }; +template <class T, class U> struct has_trivial_copy<std::pair<T, U> > + : integral_constant<bool, + (has_trivial_copy<T>::value && + has_trivial_copy<U>::value)> { }; +template <class A, int N> struct has_trivial_copy<A[N]> + : has_trivial_copy<A> { }; +template <class T> struct has_trivial_copy<const T> : has_trivial_copy<T> { }; + +// We can't get has_trivial_assign right without compiler help, so fail +// conservatively. We will assume it's false except for: (1) types +// for which is_pod is true. (2) std::pair of types with trivial copy +// constructors. (3) array of a type with a trivial assign constructor. +template <class T> struct has_trivial_assign : is_pod<T> { }; +template <class T, class U> struct has_trivial_assign<std::pair<T, U> > + : integral_constant<bool, + (has_trivial_assign<T>::value && + has_trivial_assign<U>::value)> { }; +template <class A, int N> struct has_trivial_assign<A[N]> + : has_trivial_assign<A> { }; + +// We can't get has_trivial_destructor right without compiler help, so +// fail conservatively. We will assume it's false except for: (1) types +// for which is_pod is true. (2) std::pair of types with trivial +// destructors. (3) array of a type with a trivial destructor. +// (4) const versions thereof. +template <class T> struct has_trivial_destructor : is_pod<T> { }; +template <class T, class U> struct has_trivial_destructor<std::pair<T, U> > + : integral_constant<bool, + (has_trivial_destructor<T>::value && + has_trivial_destructor<U>::value)> { }; +template <class A, int N> struct has_trivial_destructor<A[N]> + : has_trivial_destructor<A> { }; +template <class T> struct has_trivial_destructor<const T> + : has_trivial_destructor<T> { }; + +// Specified by TR1 [4.7.1] +template<typename T> struct remove_const { typedef T type; }; +template<typename T> struct remove_const<T const> { typedef T type; }; +template<typename T> struct remove_volatile { typedef T type; }; +template<typename T> struct remove_volatile<T volatile> { typedef T type; }; +template<typename T> struct remove_cv { + typedef typename remove_const<typename remove_volatile<T>::type>::type type; +}; + + +// Specified by TR1 [4.7.2] Reference modifications. +template<typename T> struct remove_reference { typedef T type; }; +template<typename T> struct remove_reference<T&> { typedef T type; }; + +template <typename T> struct add_reference { typedef T& type; }; +template <typename T> struct add_reference<T&> { typedef T& type; }; + +// Specified by TR1 [4.7.4] Pointer modifications. +template<typename T> struct remove_pointer { typedef T type; }; +template<typename T> struct remove_pointer<T*> { typedef T type; }; +template<typename T> struct remove_pointer<T* const> { typedef T type; }; +template<typename T> struct remove_pointer<T* volatile> { typedef T type; }; +template<typename T> struct remove_pointer<T* const volatile> { + typedef T type; }; + +// Specified by TR1 [4.6] Relationships between types +template<typename T, typename U> struct is_same : public false_type { }; +template<typename T> struct is_same<T, T> : public true_type { }; + +// Specified by TR1 [4.6] Relationships between types +#if !defined(_MSC_VER) && !(defined(__GNUC__) && __GNUC__ <= 3) +namespace internal { + +// This class is an implementation detail for is_convertible, and you +// don't need to know how it works to use is_convertible. For those +// who care: we declare two different functions, one whose argument is +// of type To and one with a variadic argument list. We give them +// return types of different size, so we can use sizeof to trick the +// compiler into telling us which function it would have chosen if we +// had called it with an argument of type From. See Alexandrescu's +// _Modern C++ Design_ for more details on this sort of trick. + +template <typename From, typename To> +struct ConvertHelper { + static small_ Test(To); + static big_ Test(...); + static From Create(); +}; +} // namespace internal + +// Inherits from true_type if From is convertible to To, false_type otherwise. +template <typename From, typename To> +struct is_convertible + : integral_constant<bool, + sizeof(internal::ConvertHelper<From, To>::Test( + internal::ConvertHelper<From, To>::Create())) + == sizeof(small_)> { +}; +#endif + +_END_GOOGLE_NAMESPACE_ + +// Right now these macros are no-ops, and mostly just document the fact +// these types are PODs, for human use. They may be made more contentful +// later. The typedef is just to make it legal to put a semicolon after +// these macros. +#define DECLARE_POD(TypeName) typedef int Dummy_Type_For_DECLARE_POD +#define DECLARE_NESTED_POD(TypeName) DECLARE_POD(TypeName) +#define PROPAGATE_POD_FROM_TEMPLATE_ARGUMENT(TemplateName) \ + typedef int Dummy_Type_For_PROPAGATE_POD_FROM_TEMPLATE_ARGUMENT +#define ENFORCE_POD(TypeName) typedef int Dummy_Type_For_ENFORCE_POD + +#endif // BASE_TYPE_TRAITS_H_ diff --git a/contrib/libs/sparsehash/ya.make b/contrib/libs/sparsehash/ya.make index 15ba31eb9d..d12785158a 100644 --- a/contrib/libs/sparsehash/ya.make +++ b/contrib/libs/sparsehash/ya.make @@ -1,7 +1,7 @@ # Generated by devtools/yamaker from nixpkgs 930da485d9af8100f8858bd6fe8f28e3eca26933. -LIBRARY() - +LIBRARY() + LICENSE(BSD-3-Clause) LICENSE_TEXTS(.yandex_meta/licenses.list.txt) @@ -13,5 +13,5 @@ VERSION(2.0.4) ADDINCL( GLOBAL contrib/libs/sparsehash/src ) - -END() + +END() diff --git a/library/cpp/actors/dnscachelib/timekeeper.h b/library/cpp/actors/dnscachelib/timekeeper.h index d8dc5560bc..0528d8549c 100644 --- a/library/cpp/actors/dnscachelib/timekeeper.h +++ b/library/cpp/actors/dnscachelib/timekeeper.h @@ -4,7 +4,7 @@ #include <util/generic/singleton.h> #include <util/string/cast.h> #include <util/system/thread.h> -#include <util/system/event.h> +#include <util/system/event.h> #include <util/system/env.h> #include <cstdlib> @@ -59,11 +59,11 @@ private: static void* Worker(void* arg) { TTimeKeeper* owner = static_cast<TTimeKeeper*>(arg); - do { + do { /* Race condition may occur here but locking looks too expensive */ gettimeofday(&owner->CurrentTime, nullptr); - } while (!owner->Exit.WaitT(TDuration::MicroSeconds(UpdateInterval))); + } while (!owner->Exit.WaitT(TDuration::MicroSeconds(UpdateInterval))); return nullptr; } diff --git a/library/cpp/balloc/malloc-info.cpp b/library/cpp/balloc/malloc-info.cpp index de48ae41ce..604b1fb145 100644 --- a/library/cpp/balloc/malloc-info.cpp +++ b/library/cpp/balloc/malloc-info.cpp @@ -1,13 +1,13 @@ #include <library/cpp/malloc/api/malloc.h> - + #include <string.h> - -using namespace NMalloc; - + +using namespace NMalloc; + extern "C" void DisableBalloc(); extern "C" void EnableBalloc(); extern "C" bool BallocDisabled(); - + namespace { bool SetAllocParam(const char* name, const char* value) { if (strcmp(name, "disable") == 0) { @@ -30,12 +30,12 @@ namespace { } } -TMallocInfo NMalloc::MallocInfo() { - TMallocInfo r; +TMallocInfo NMalloc::MallocInfo() { + TMallocInfo r; - r.Name = "balloc"; + r.Name = "balloc"; r.SetParam = SetAllocParam; r.CheckParam = CheckAllocParam; - return r; -} + return r; +} diff --git a/library/cpp/cgiparam/cgiparam_ut.cpp b/library/cpp/cgiparam/cgiparam_ut.cpp index e33efa3f26..a562342084 100644 --- a/library/cpp/cgiparam/cgiparam_ut.cpp +++ b/library/cpp/cgiparam/cgiparam_ut.cpp @@ -12,9 +12,9 @@ Y_UNIT_TEST_SUITE(TCgiParametersTest) { UNIT_ASSERT(C.Has("ag0", "")); UNIT_ASSERT(!C.Has("a", "bbb")); UNIT_ASSERT(!C.Has("aaa", "bb")); - - UNIT_ASSERT(C.Has("ccc")); - UNIT_ASSERT(!C.Has("zzzzzz")); + + UNIT_ASSERT(C.Has("ccc")); + UNIT_ASSERT(!C.Has("zzzzzz")); } Y_UNIT_TEST(TestQuick) { diff --git a/library/cpp/containers/compact_vector/compact_vector.h b/library/cpp/containers/compact_vector/compact_vector.h index 0e9667c906..dbe7473f0c 100644 --- a/library/cpp/containers/compact_vector/compact_vector.h +++ b/library/cpp/containers/compact_vector/compact_vector.h @@ -1,209 +1,209 @@ #pragma once - -#include <util/generic/yexception.h> -#include <util/generic/utility.h> + +#include <util/generic/yexception.h> +#include <util/generic/utility.h> #include <util/memory/alloc.h> #include <util/stream/output.h> -#include <util/system/yassert.h> - +#include <util/system/yassert.h> + #include <cstdlib> // vector that is 8 bytes when empty (TVector is 24 bytes) - -template <typename T> -class TCompactVector { -private: - typedef TCompactVector<T> TThis; - - // XXX: make header independent on T and introduce nullptr - struct THeader { - size_t Size; - size_t Capacity; - }; - - T* Ptr; - - THeader* Header() { + +template <typename T> +class TCompactVector { +private: + typedef TCompactVector<T> TThis; + + // XXX: make header independent on T and introduce nullptr + struct THeader { + size_t Size; + size_t Capacity; + }; + + T* Ptr; + + THeader* Header() { return ((THeader*)Ptr) - 1; - } - - const THeader* Header() const { + } + + const THeader* Header() const { return ((THeader*)Ptr) - 1; - } - -public: - typedef T* TIterator; - typedef const T* TConstIterator; - - typedef TIterator iterator; - typedef TConstIterator const_iterator; - - TCompactVector() + } + +public: + typedef T* TIterator; + typedef const T* TConstIterator; + + typedef TIterator iterator; + typedef TConstIterator const_iterator; + + TCompactVector() : Ptr(nullptr) { } - - TCompactVector(const TThis& that) + + TCompactVector(const TThis& that) : Ptr(nullptr) - { - Reserve(that.Size()); - for (TConstIterator i = that.Begin(); i != that.End(); ++i) { - PushBack(*i); - } - } - - ~TCompactVector() { - for (size_t i = 0; i < Size(); ++i) { - try { - (*this)[i].~T(); - } catch (...) { - } - } + { + Reserve(that.Size()); + for (TConstIterator i = that.Begin(); i != that.End(); ++i) { + PushBack(*i); + } + } + + ~TCompactVector() { + for (size_t i = 0; i < Size(); ++i) { + try { + (*this)[i].~T(); + } catch (...) { + } + } if (Ptr) - free(Header()); - } - - TIterator Begin() { - return Ptr; - } - - TIterator End() { - return Ptr + Size(); - } - - TConstIterator Begin() const { - return Ptr; - } - - TConstIterator End() const { - return Ptr + Size(); - } - - iterator begin() { - return Begin(); - } - - const_iterator begin() const { - return Begin(); - } - - iterator end() { - return End(); - } - - const_iterator end() const { - return End(); - } - - void Swap(TThis& that) { - DoSwap(Ptr, that.Ptr); - } - - void Reserve(size_t newCapacity) { - if (newCapacity <= Capacity()) { + free(Header()); + } + + TIterator Begin() { + return Ptr; + } + + TIterator End() { + return Ptr + Size(); + } + + TConstIterator Begin() const { + return Ptr; + } + + TConstIterator End() const { + return Ptr + Size(); + } + + iterator begin() { + return Begin(); + } + + const_iterator begin() const { + return Begin(); + } + + iterator end() { + return End(); + } + + const_iterator end() const { + return End(); + } + + void Swap(TThis& that) { + DoSwap(Ptr, that.Ptr); + } + + void Reserve(size_t newCapacity) { + if (newCapacity <= Capacity()) { } else if (Ptr == nullptr) { void* mem = ::malloc(sizeof(THeader) + newCapacity * sizeof(T)); if (mem == nullptr) - ythrow yexception() << "out of memory"; + ythrow yexception() << "out of memory"; Ptr = (T*)(((THeader*)mem) + 1); - Header()->Size = 0; - Header()->Capacity = newCapacity; - } else { - TThis copy; - size_t realNewCapacity = Max(Capacity() * 2, newCapacity); - copy.Reserve(realNewCapacity); - for (TConstIterator it = Begin(); it != End(); ++it) { - copy.PushBack(*it); - } - Swap(copy); - } - } - - size_t Size() const { + Header()->Size = 0; + Header()->Capacity = newCapacity; + } else { + TThis copy; + size_t realNewCapacity = Max(Capacity() * 2, newCapacity); + copy.Reserve(realNewCapacity); + for (TConstIterator it = Begin(); it != End(); ++it) { + copy.PushBack(*it); + } + Swap(copy); + } + } + + size_t Size() const { return Ptr ? Header()->Size : 0; - } - - size_t size() const { - return Size(); - } - - bool Empty() const { - return Size() == 0; - } - - bool empty() const { - return Empty(); - } - - size_t Capacity() const { + } + + size_t size() const { + return Size(); + } + + bool Empty() const { + return Size() == 0; + } + + bool empty() const { + return Empty(); + } + + size_t Capacity() const { return Ptr ? Header()->Capacity : 0; - } - - void PushBack(const T& elem) { - Reserve(Size() + 1); - new (Ptr + Size()) T(elem); - ++(Header()->Size); - } - - T& Back() { - return *(End() - 1); - } - - const T& Back() const { - return *(End() - 1); - } - - T& back() { - return Back(); - } - - const T& back() const { - return Back(); - } - - TIterator Insert(TIterator pos, const T& elem) { + } + + void PushBack(const T& elem) { + Reserve(Size() + 1); + new (Ptr + Size()) T(elem); + ++(Header()->Size); + } + + T& Back() { + return *(End() - 1); + } + + const T& Back() const { + return *(End() - 1); + } + + T& back() { + return Back(); + } + + const T& back() const { + return Back(); + } + + TIterator Insert(TIterator pos, const T& elem) { Y_ASSERT(pos >= Begin()); Y_ASSERT(pos <= End()); - - size_t posn = pos - Begin(); - if (pos == End()) { - PushBack(elem); - } else { + + size_t posn = pos - Begin(); + if (pos == End()) { + PushBack(elem); + } else { Y_ASSERT(Size() > 0); - - Reserve(Size() + 1); - - PushBack(*(End() - 1)); - - for (size_t i = Size() - 2; i + 1 > posn; --i) { - (*this)[i + 1] = (*this)[i]; - } - - (*this)[posn] = elem; - } - return Begin() + posn; - } - - iterator insert(iterator pos, const T& elem) { - return Insert(pos, elem); - } - - void Clear() { - TThis clean; - Swap(clean); - } - - void clear() { - Clear(); - } - + + Reserve(Size() + 1); + + PushBack(*(End() - 1)); + + for (size_t i = Size() - 2; i + 1 > posn; --i) { + (*this)[i + 1] = (*this)[i]; + } + + (*this)[posn] = elem; + } + return Begin() + posn; + } + + iterator insert(iterator pos, const T& elem) { + return Insert(pos, elem); + } + + void Clear() { + TThis clean; + Swap(clean); + } + + void clear() { + Clear(); + } + T& operator[](size_t index) { Y_ASSERT(index < Size()); - return Ptr[index]; - } - + return Ptr[index]; + } + const T& operator[](size_t index) const { Y_ASSERT(index < Size()); - return Ptr[index]; - } -}; + return Ptr[index]; + } +}; diff --git a/library/cpp/containers/compact_vector/compact_vector_ut.cpp b/library/cpp/containers/compact_vector/compact_vector_ut.cpp index cb5e603fd9..7d413d6575 100644 --- a/library/cpp/containers/compact_vector/compact_vector_ut.cpp +++ b/library/cpp/containers/compact_vector/compact_vector_ut.cpp @@ -1,39 +1,39 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "compact_vector.h" - + +#include "compact_vector.h" + Y_UNIT_TEST_SUITE(TCompactVectorTest) { Y_UNIT_TEST(TestSimple1) { - } - + } + Y_UNIT_TEST(TestSimple) { - TCompactVector<ui32> vector; - for (ui32 i = 0; i < 10000; ++i) { - vector.PushBack(i + 20); - UNIT_ASSERT_VALUES_EQUAL(i + 1, vector.Size()); - } - for (ui32 i = 0; i < 10000; ++i) { - UNIT_ASSERT_VALUES_EQUAL(i + 20, vector[i]); - } - } - + TCompactVector<ui32> vector; + for (ui32 i = 0; i < 10000; ++i) { + vector.PushBack(i + 20); + UNIT_ASSERT_VALUES_EQUAL(i + 1, vector.Size()); + } + for (ui32 i = 0; i < 10000; ++i) { + UNIT_ASSERT_VALUES_EQUAL(i + 20, vector[i]); + } + } + Y_UNIT_TEST(TestInsert) { - TCompactVector<ui32> vector; - - for (ui32 i = 0; i < 10; ++i) { - vector.PushBack(i + 2); - } - - vector.Insert(vector.Begin(), 99); - + TCompactVector<ui32> vector; + + for (ui32 i = 0; i < 10; ++i) { + vector.PushBack(i + 2); + } + + vector.Insert(vector.Begin(), 99); + UNIT_ASSERT_VALUES_EQUAL(11u, vector.Size()); UNIT_ASSERT_VALUES_EQUAL(99u, vector[0]); - for (ui32 i = 0; i < 10; ++i) { - UNIT_ASSERT_VALUES_EQUAL(i + 2, vector[i + 1]); - } - - vector.Insert(vector.Begin() + 3, 77); - + for (ui32 i = 0; i < 10; ++i) { + UNIT_ASSERT_VALUES_EQUAL(i + 2, vector[i + 1]); + } + + vector.Insert(vector.Begin() + 3, 77); + UNIT_ASSERT_VALUES_EQUAL(12u, vector.Size()); UNIT_ASSERT_VALUES_EQUAL(99u, vector[0]); UNIT_ASSERT_VALUES_EQUAL(2u, vector[1]); @@ -42,5 +42,5 @@ Y_UNIT_TEST_SUITE(TCompactVectorTest) { UNIT_ASSERT_VALUES_EQUAL(4u, vector[4]); UNIT_ASSERT_VALUES_EQUAL(5u, vector[5]); UNIT_ASSERT_VALUES_EQUAL(11u, vector[11]); - } -} + } +} diff --git a/library/cpp/containers/comptrie/array_with_size.h b/library/cpp/containers/comptrie/array_with_size.h index 4a1c85ce68..36e61c7410 100644 --- a/library/cpp/containers/comptrie/array_with_size.h +++ b/library/cpp/containers/comptrie/array_with_size.h @@ -1,67 +1,67 @@ #pragma once - -#include <util/generic/ptr.h> -#include <util/generic/noncopyable.h> -#include <util/generic/utility.h> + +#include <util/generic/ptr.h> +#include <util/generic/noncopyable.h> +#include <util/generic/utility.h> #include <util/system/sys_alloc.h> - -template <typename T> + +template <typename T> class TArrayWithSizeHolder : TNonCopyable { - typedef TArrayWithSizeHolder<T> TThis; - - T* Data; - -public: + typedef TArrayWithSizeHolder<T> TThis; + + T* Data; + +public: TArrayWithSizeHolder() : Data(nullptr) { } - - ~TArrayWithSizeHolder() { - if (!Data) - return; - for (size_t i = 0; i < Size(); ++i) { - try { - Data[i].~T(); - } catch (...) { - } - } + + ~TArrayWithSizeHolder() { + if (!Data) + return; + for (size_t i = 0; i < Size(); ++i) { + try { + Data[i].~T(); + } catch (...) { + } + } y_deallocate(((size_t*)Data) - 1); - } - - void Swap(TThis& copy) { - DoSwap(Data, copy.Data); - } - - void Resize(size_t newSize) { - if (newSize == Size()) - return; - TThis copy; + } + + void Swap(TThis& copy) { + DoSwap(Data, copy.Data); + } + + void Resize(size_t newSize) { + if (newSize == Size()) + return; + TThis copy; copy.Data = (T*)(((size_t*)y_allocate(sizeof(size_t) + sizeof(T) * newSize)) + 1); - // does not handle constructor exceptions properly - for (size_t i = 0; i < Min(Size(), newSize); ++i) { - new (copy.Data + i) T(Data[i]); - } - for (size_t i = Min(Size(), newSize); i < newSize; ++i) { - new (copy.Data + i) T; - } + // does not handle constructor exceptions properly + for (size_t i = 0; i < Min(Size(), newSize); ++i) { + new (copy.Data + i) T(Data[i]); + } + for (size_t i = Min(Size(), newSize); i < newSize; ++i) { + new (copy.Data + i) T; + } ((size_t*)copy.Data)[-1] = newSize; - Swap(copy); - } - - size_t Size() const { + Swap(copy); + } + + size_t Size() const { return Data ? ((size_t*)Data)[-1] : 0; - } - - bool Empty() const { - return Size() == 0; - } - - T* Get() { - return Data; - } - - const T* Get() const { - return Data; - } -}; + } + + bool Empty() const { + return Size() == 0; + } + + T* Get() { + return Data; + } + + const T* Get() const { + return Data; + } +}; diff --git a/library/cpp/containers/comptrie/comptrie_builder.h b/library/cpp/containers/comptrie/comptrie_builder.h index 652c3151c5..cf7d2e39a3 100644 --- a/library/cpp/containers/comptrie/comptrie_builder.h +++ b/library/cpp/containers/comptrie/comptrie_builder.h @@ -53,19 +53,19 @@ public: bool Add(const TSymbol* key, size_t keylen, const TData& value); bool Add(const TKeyBuf& key, const TData& value) { return Add(key.data(), key.size(), value); - } + } - // add already serialized data + // add already serialized data bool AddPtr(const TSymbol* key, size_t keylen, const char* data); bool AddPtr(const TKeyBuf& key, const char* data) { return AddPtr(key.data(), key.size(), data); - } - + } + bool AddSubtreeInFile(const TSymbol* key, size_t keylen, const TString& filename); bool AddSubtreeInFile(const TKeyBuf& key, const TString& filename) { return AddSubtreeInFile(key.data(), key.size(), filename); - } - + } + bool AddSubtreeInBuffer(const TSymbol* key, size_t keylen, TArrayWithSizeHolder<char>&& buffer); bool AddSubtreeInBuffer(const TKeyBuf& key, TArrayWithSizeHolder<char>&& buffer) { return AddSubtreeInBuffer(key.data(), key.size(), std::move(buffer)); @@ -74,8 +74,8 @@ public: bool Find(const TSymbol* key, size_t keylen, TData* value) const; bool Find(const TKeyBuf& key, TData* value = nullptr) const { return Find(key.data(), key.size(), value); - } - + } + bool FindLongestPrefix(const TSymbol* key, size_t keylen, size_t* prefixLen, TData* value = nullptr) const; bool FindLongestPrefix(const TKeyBuf& key, size_t* prefixLen, TData* value = nullptr) const { return FindLongestPrefix(key.data(), key.size(), prefixLen, value); @@ -85,8 +85,8 @@ public: size_t SaveAndDestroy(IOutputStream& os); size_t SaveToFile(const TString& fileName) const { TFixedBufferFileOutput out(fileName); - return Save(out); - } + return Save(out); + } void Clear(); // Returns all memory to the system and resets the builder state. diff --git a/library/cpp/containers/comptrie/comptrie_builder.inl b/library/cpp/containers/comptrie/comptrie_builder.inl index 1410d4590f..f273fa6571 100644 --- a/library/cpp/containers/comptrie/comptrie_builder.inl +++ b/library/cpp/containers/comptrie/comptrie_builder.inl @@ -28,54 +28,54 @@ template <class T, class D, class S> class TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl { protected: TMemoryPool Pool; - size_t PayloadSize; - THolder<TFixedSizeAllocator> NodeAllocator; + size_t PayloadSize; + THolder<TFixedSizeAllocator> NodeAllocator; class TNode; - class TArc; + class TArc; TNode* Root; TCompactTrieBuilderFlags Flags; size_t EntryCount; size_t NodeCount; - TPacker Packer; - - enum EPayload { - DATA_ABSENT, - DATA_INSIDE, - DATA_MALLOCED, - DATA_IN_MEMPOOL, - }; - + TPacker Packer; + + enum EPayload { + DATA_ABSENT, + DATA_INSIDE, + DATA_MALLOCED, + DATA_IN_MEMPOOL, + }; + protected: void ConvertSymbolArrayToChar(const TSymbol* key, size_t keylen, TTempBuf& buf, size_t ckeylen) const; - void NodeLinkTo(TNode* thiz, const TBlob& label, TNode* node); + void NodeLinkTo(TNode* thiz, const TBlob& label, TNode* node); TNode* NodeForwardAdd(TNode* thiz, const char* label, size_t len, size_t& passed, size_t* nodeCount); bool FindEntryImpl(const char* key, size_t keylen, TData* value) const; bool FindLongestPrefixImpl(const char* keyptr, size_t keylen, size_t* prefixLen, TData* value) const; - size_t NodeMeasureSubtree(TNode* thiz) const; + size_t NodeMeasureSubtree(TNode* thiz) const; ui64 NodeSaveSubtree(TNode* thiz, IOutputStream& os) const; ui64 NodeSaveSubtreeAndDestroy(TNode* thiz, IOutputStream& osy); - void NodeBufferSubtree(TNode* thiz); - - size_t NodeMeasureLeafValue(TNode* thiz) const; + void NodeBufferSubtree(TNode* thiz); + + size_t NodeMeasureLeafValue(TNode* thiz) const; ui64 NodeSaveLeafValue(TNode* thiz, IOutputStream& os) const; - + virtual ui64 ArcMeasure(const TArc* thiz, size_t leftsize, size_t rightsize) const; - + virtual ui64 ArcSaveSelf(const TArc* thiz, IOutputStream& os) const; ui64 ArcSave(const TArc* thiz, IOutputStream& os) const; ui64 ArcSaveAndDestroy(const TArc* thiz, IOutputStream& os); - + public: TCompactTrieBuilderImpl(TCompactTrieBuilderFlags flags, TPacker packer, IAllocator* alloc); virtual ~TCompactTrieBuilderImpl(); - void DestroyNode(TNode* node); - void NodeReleasePayload(TNode* thiz); - + void DestroyNode(TNode* node); + void NodeReleasePayload(TNode* thiz); + char* AddEntryForData(const TSymbol* key, size_t keylen, size_t dataLen, bool& isNewAddition); TNode* AddEntryForSomething(const TSymbol* key, size_t keylen, bool& isNewAddition); - + bool AddEntry(const TSymbol* key, size_t keylen, const TData& value); bool AddEntryPtr(const TSymbol* key, size_t keylen, const char* value); bool AddSubtreeInFile(const TSymbol* key, size_t keylen, const TString& fileName); @@ -98,38 +98,38 @@ public: }; template <class T, class D, class S> -class TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TArc { +class TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TArc { public: - TBlob Label; - TNode* Node; - mutable size_t LeftOffset; - mutable size_t RightOffset; + TBlob Label; + TNode* Node; + mutable size_t LeftOffset; + mutable size_t RightOffset; - TArc(const TBlob& lbl, TNode* nd); -}; + TArc(const TBlob& lbl, TNode* nd); +}; -template <class T, class D, class S> -class TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode { -public: - typedef typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl TBuilderImpl; - typedef typename TBuilderImpl::TArc TArc; +template <class T, class D, class S> +class TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode { +public: + typedef typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl TBuilderImpl; + typedef typename TBuilderImpl::TArc TArc; - struct ISubtree { + struct ISubtree { virtual ~ISubtree() = default; - virtual bool IsLast() const = 0; - virtual ui64 Measure(const TBuilderImpl* builder) const = 0; + virtual bool IsLast() const = 0; + virtual ui64 Measure(const TBuilderImpl* builder) const = 0; virtual ui64 Save(const TBuilderImpl* builder, IOutputStream& os) const = 0; virtual ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) = 0; - virtual void Destroy(TBuilderImpl*) { } + virtual void Destroy(TBuilderImpl*) { } // Tries to find key in subtree. // Returns next node to find the key in (to avoid recursive calls) // If it has end result, writes it to @value and @result arguments and returns nullptr virtual const TNode* Find(TStringBuf& key, TData* value, bool& result, const TPacker& packer) const = 0; virtual const TNode* FindLongestPrefix(TStringBuf& key, TData* value, bool& result, const TPacker& packer) const = 0; - }; - - class TArcSet: public ISubtree, public TCompactVector<TArc> { + }; + + class TArcSet: public ISubtree, public TCompactVector<TArc> { public: typedef typename TCompactVector<TArc>::iterator iterator; typedef typename TCompactVector<TArc>::const_iterator const_iterator; @@ -141,35 +141,35 @@ public: iterator Find(char ch); const_iterator Find(char ch) const; void Add(const TBlob& s, TNode* node); - + bool IsLast() const override { - return this->Empty(); - } - + return this->Empty(); + } + const TNode* Find(TStringBuf& key, TData* value, bool& result, const TPacker& packer) const override; const TNode* FindLongestPrefix(TStringBuf& key, TData* value, bool& result, const TPacker& packer) const override { return Find(key, value, result, packer); } ui64 Measure(const TBuilderImpl* builder) const override { - return MeasureRange(builder, 0, this->size()); - } - - ui64 MeasureRange(const TBuilderImpl* builder, size_t from, size_t to) const { - if (from >= to) - return 0; - - size_t median = (from + to) / 2; + return MeasureRange(builder, 0, this->size()); + } + + ui64 MeasureRange(const TBuilderImpl* builder, size_t from, size_t to) const { + if (from >= to) + return 0; + + size_t median = (from + to) / 2; size_t leftsize = (size_t)MeasureRange(builder, from, median); size_t rightsize = (size_t)MeasureRange(builder, median + 1, to); - - return builder->ArcMeasure(&(*this)[median], leftsize, rightsize); - } - + + return builder->ArcMeasure(&(*this)[median], leftsize, rightsize); + } + ui64 Save(const TBuilderImpl* builder, IOutputStream& os) const override { - return SaveRange(builder, 0, this->size(), os); - } - + return SaveRange(builder, 0, this->size(), os); + } + ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) override { ui64 result = SaveRangeAndDestroy(builder, 0, this->size(), os); Destroy(builder); @@ -177,17 +177,17 @@ public: } ui64 SaveRange(const TBuilderImpl* builder, size_t from, size_t to, IOutputStream& os) const { - if (from >= to) - return 0; - - size_t median = (from + to) / 2; - - ui64 written = builder->ArcSave(&(*this)[median], os); - written += SaveRange(builder, from, median, os); - written += SaveRange(builder, median + 1, to, os); - return written; - } - + if (from >= to) + return 0; + + size_t median = (from + to) / 2; + + ui64 written = builder->ArcSave(&(*this)[median], os); + written += SaveRange(builder, from, median, os); + written += SaveRange(builder, median + 1, to, os); + return written; + } + ui64 SaveRangeAndDestroy(TBuilderImpl* builder, size_t from, size_t to, IOutputStream& os) { if (from >= to) return 0; @@ -201,30 +201,30 @@ public: } void Destroy(TBuilderImpl* builder) override { - // Delete all nodes down the stream. - for (iterator it = this->begin(); it != this->end(); ++it) { - builder->DestroyNode(it->Node); - } - this->clear(); - } - + // Delete all nodes down the stream. + for (iterator it = this->begin(); it != this->end(); ++it) { + builder->DestroyNode(it->Node); + } + this->clear(); + } + ~TArcSet() override { Y_ASSERT(this->empty()); - } - + } + }; - struct TBufferedSubtree: public ISubtree { - TArrayWithSizeHolder<char> Buffer; - + struct TBufferedSubtree: public ISubtree { + TArrayWithSizeHolder<char> Buffer; + TBufferedSubtree() { Y_ASSERT(reinterpret_cast<ISubtree*>(this) == static_cast<void*>(this)); // This assumption is used in TNode::Subtree() } bool IsLast() const override { - return Buffer.Empty(); - } - + return Buffer.Empty(); + } + const TNode* Find(TStringBuf& key, TData* value, bool& result, const TPacker& packer) const override { if (Buffer.Empty()) { result = false; @@ -252,45 +252,45 @@ public: } ui64 Measure(const TBuilderImpl*) const override { - return Buffer.Size(); - } - + return Buffer.Size(); + } + ui64 Save(const TBuilderImpl*, IOutputStream& os) const override { - os.Write(Buffer.Get(), Buffer.Size()); - return Buffer.Size(); - } + os.Write(Buffer.Get(), Buffer.Size()); + return Buffer.Size(); + } ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) override { ui64 result = Save(builder, os); TArrayWithSizeHolder<char>().Swap(Buffer); return result; } - }; - - struct TSubtreeInFile: public ISubtree { - struct TData { + }; + + struct TSubtreeInFile: public ISubtree { + struct TData { TString FileName; - ui64 Size; - }; - THolder<TData> Data; - + ui64 Size; + }; + THolder<TData> Data; + TSubtreeInFile(const TString& fileName) { - // stupid API - TFile file(fileName, RdOnly); - i64 size = file.GetLength(); - if (size < 0) - ythrow yexception() << "unable to get file " << fileName.Quote() << " size for unknown reason"; - Data.Reset(new TData); - Data->FileName = fileName; - Data->Size = size; + // stupid API + TFile file(fileName, RdOnly); + i64 size = file.GetLength(); + if (size < 0) + ythrow yexception() << "unable to get file " << fileName.Quote() << " size for unknown reason"; + Data.Reset(new TData); + Data->FileName = fileName; + Data->Size = size; Y_ASSERT(reinterpret_cast<ISubtree*>(this) == static_cast<void*>(this)); // This assumption is used in TNode::Subtree() - } - + } + bool IsLast() const override { - return Data->Size == 0; - } - + return Data->Size == 0; + } + const TNode* Find(TStringBuf& key, typename TCompactTrieBuilder::TData* value, bool& result, const TPacker& packer) const override { if (!Data) { result = false; @@ -317,104 +317,104 @@ public: } ui64 Measure(const TBuilderImpl*) const override { - return Data->Size; - } - + return Data->Size; + } + ui64 Save(const TBuilderImpl*, IOutputStream& os) const override { TUnbufferedFileInput is(Data->FileName); - ui64 written = TransferData(&is, &os); - if (written != Data->Size) - ythrow yexception() << "file " << Data->FileName.Quote() << " size changed"; - return written; - } + ui64 written = TransferData(&is, &os); + if (written != Data->Size) + ythrow yexception() << "file " << Data->FileName.Quote() << " size changed"; + return written; + } ui64 SaveAndDestroy(TBuilderImpl* builder, IOutputStream& os) override { return Save(builder, os); } - }; - - union { + }; + + union { char ArcsData[CONSTEXPR_MAX3(sizeof(TArcSet), sizeof(TBufferedSubtree), sizeof(TSubtreeInFile))]; union { void* Data1; long long int Data2; } Aligner; - }; - - inline ISubtree* Subtree() { - return reinterpret_cast<ISubtree*>(ArcsData); - } - - inline const ISubtree* Subtree() const { - return reinterpret_cast<const ISubtree*>(ArcsData); - } - - EPayload PayloadType; + }; + + inline ISubtree* Subtree() { + return reinterpret_cast<ISubtree*>(ArcsData); + } + + inline const ISubtree* Subtree() const { + return reinterpret_cast<const ISubtree*>(ArcsData); + } + + EPayload PayloadType; inline const char* PayloadPtr() const { return ((const char*) this) + sizeof(TNode); } - inline char* PayloadPtr() { - return ((char*) this) + sizeof(TNode); - } - - // *Payload() + inline char* PayloadPtr() { + return ((char*) this) + sizeof(TNode); + } + + // *Payload() inline const char*& PayloadAsPtr() const { const char** payload = (const char**) PayloadPtr(); return *payload; } - inline char*& PayloadAsPtr() { - char** payload = (char**) PayloadPtr(); - return *payload; - } - + inline char*& PayloadAsPtr() { + char** payload = (char**) PayloadPtr(); + return *payload; + } + inline const char* GetPayload() const { - switch (PayloadType) { - case DATA_INSIDE: - return PayloadPtr(); - case DATA_MALLOCED: - case DATA_IN_MEMPOOL: - return PayloadAsPtr(); + switch (PayloadType) { + case DATA_INSIDE: + return PayloadPtr(); + case DATA_MALLOCED: + case DATA_IN_MEMPOOL: + return PayloadAsPtr(); case DATA_ABSENT: - default: - abort(); - } - } - + default: + abort(); + } + } + inline char* GetPayload() { const TNode* thiz = this; return const_cast<char*>(thiz->GetPayload()); // const_cast is to avoid copy-paste style } - bool IsFinal() const { - return PayloadType != DATA_ABSENT; - } - - bool IsLast() const { - return Subtree()->IsLast(); - } - - inline void* operator new(size_t, TFixedSizeAllocator& pool) { - return pool.Allocate(); - } - + bool IsFinal() const { + return PayloadType != DATA_ABSENT; + } + + bool IsLast() const { + return Subtree()->IsLast(); + } + + inline void* operator new(size_t, TFixedSizeAllocator& pool) { + return pool.Allocate(); + } + inline void operator delete(void* ptr, TFixedSizeAllocator& pool) noexcept { - pool.Release(ptr); - } - - TNode() - : PayloadType(DATA_ABSENT) - { - new (Subtree()) TArcSet; - } - - ~TNode() { - Subtree()->~ISubtree(); + pool.Release(ptr); + } + + TNode() + : PayloadType(DATA_ABSENT) + { + new (Subtree()) TArcSet; + } + + ~TNode() { + Subtree()->~ISubtree(); Y_ASSERT(PayloadType == DATA_ABSENT); - } - + } + }; // TCompactTrieBuilder @@ -433,14 +433,14 @@ bool TCompactTrieBuilder<T, D, S>::Add(const TSymbol* key, size_t keylen, const template <class T, class D, class S> bool TCompactTrieBuilder<T, D, S>::AddPtr(const TSymbol* key, size_t keylen, const char* value) { return Impl->AddEntryPtr(key, keylen, value); -} - -template <class T, class D, class S> +} + +template <class T, class D, class S> bool TCompactTrieBuilder<T, D, S>::AddSubtreeInFile(const TSymbol* key, size_t keylen, const TString& fileName) { return Impl->AddSubtreeInFile(key, keylen, fileName); -} - -template <class T, class D, class S> +} + +template <class T, class D, class S> bool TCompactTrieBuilder<T, D, S>::AddSubtreeInBuffer(const TSymbol* key, size_t keylen, TArrayWithSizeHolder<char>&& buffer) { return Impl->AddSubtreeInBuffer(key, keylen, std::move(buffer)); } @@ -486,19 +486,19 @@ size_t TCompactTrieBuilder<T, D, S>::GetNodeCount() const { template <class T, class D, class S> TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TCompactTrieBuilderImpl(TCompactTrieBuilderFlags flags, TPacker packer, IAllocator* alloc) : Pool(1000000, TMemoryPool::TLinearGrow::Instance(), alloc) - , PayloadSize(sizeof(void*)) // XXX: find better value + , PayloadSize(sizeof(void*)) // XXX: find better value , NodeAllocator(new TFixedSizeAllocator(sizeof(TNode) + PayloadSize, alloc)) , Flags(flags) , EntryCount(0) , NodeCount(1) - , Packer(packer) + , Packer(packer) { - Root = new (*NodeAllocator) TNode; + Root = new (*NodeAllocator) TNode; } template <class T, class D, class S> TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::~TCompactTrieBuilderImpl() { - DestroyNode(Root); + DestroyNode(Root); } template <class T, class D, class S> @@ -519,67 +519,67 @@ void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ConvertSymbolArrayTo } template <class T, class D, class S> -void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::DestroyNode(TNode* thiz) { - thiz->Subtree()->Destroy(this); - NodeReleasePayload(thiz); - thiz->~TNode(); - NodeAllocator->Release(thiz); -} - -template <class T, class D, class S> -void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeReleasePayload(TNode* thiz) { - switch (thiz->PayloadType) { - case DATA_ABSENT: - case DATA_INSIDE: - case DATA_IN_MEMPOOL: - break; - case DATA_MALLOCED: - delete[] thiz->PayloadAsPtr(); - break; - default: - abort(); - } - thiz->PayloadType = DATA_ABSENT; -} - -template <class T, class D, class S> +void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::DestroyNode(TNode* thiz) { + thiz->Subtree()->Destroy(this); + NodeReleasePayload(thiz); + thiz->~TNode(); + NodeAllocator->Release(thiz); +} + +template <class T, class D, class S> +void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeReleasePayload(TNode* thiz) { + switch (thiz->PayloadType) { + case DATA_ABSENT: + case DATA_INSIDE: + case DATA_IN_MEMPOOL: + break; + case DATA_MALLOCED: + delete[] thiz->PayloadAsPtr(); + break; + default: + abort(); + } + thiz->PayloadType = DATA_ABSENT; +} + +template <class T, class D, class S> bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::AddEntry( const TSymbol* key, size_t keylen, const TData& value) { - size_t datalen = Packer.MeasureLeaf(value); - + size_t datalen = Packer.MeasureLeaf(value); + bool isNewAddition = false; char* place = AddEntryForData(key, keylen, datalen, isNewAddition); - Packer.PackLeaf(place, value, datalen); + Packer.PackLeaf(place, value, datalen); return isNewAddition; -} - -template <class T, class D, class S> +} + +template <class T, class D, class S> bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::AddEntryPtr( const TSymbol* key, size_t keylen, const char* value) { - size_t datalen = Packer.SkipLeaf(value); - + size_t datalen = Packer.SkipLeaf(value); + bool isNewAddition = false; char* place = AddEntryForData(key, keylen, datalen, isNewAddition); - memcpy(place, value, datalen); + memcpy(place, value, datalen); return isNewAddition; -} - -template <class T, class D, class S> +} + +template <class T, class D, class S> bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::AddSubtreeInFile( const TSymbol* key, size_t keylen, const TString& fileName) { - typedef typename TNode::ISubtree ISubtree; - typedef typename TNode::TSubtreeInFile TSubtreeInFile; - + typedef typename TNode::ISubtree ISubtree; + typedef typename TNode::TSubtreeInFile TSubtreeInFile; + bool isNewAddition = false; TNode* node = AddEntryForSomething(key, keylen, isNewAddition); - node->Subtree()->Destroy(this); - node->Subtree()->~ISubtree(); - - new (node->Subtree()) TSubtreeInFile(fileName); + node->Subtree()->Destroy(this); + node->Subtree()->~ISubtree(); + + new (node->Subtree()) TSubtreeInFile(fileName); return isNewAddition; -} - -template <class T, class D, class S> +} + +template <class T, class D, class S> bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::AddSubtreeInBuffer( const TSymbol* key, size_t keylen, TArrayWithSizeHolder<char>&& buffer) { @@ -613,10 +613,10 @@ typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode* // Special case of empty key: replace it by 1-byte "\0" key. size_t ckeylen = keylen ? keylen * sizeof(TSymbol) : 1; TTempBuf ckeybuf(ckeylen); - if (keylen == 0) { + if (keylen == 0) { ckeybuf.Append("\0", 1); - } else { - ConvertSymbolArrayToChar(key, keylen, ckeybuf, ckeylen); + } else { + ConvertSymbolArrayToChar(key, keylen, ckeybuf, ckeylen); } char* ckey = ckeybuf.Data(); @@ -638,24 +638,24 @@ typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode* isNewAddition = (current->PayloadType == DATA_ABSENT); if ((Flags & CTBF_UNIQUE) && !isNewAddition) ythrow yexception() << "Duplicate key"; - return current; -} + return current; +} -template <class T, class D, class S> +template <class T, class D, class S> char* TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::AddEntryForData(const TSymbol* key, size_t keylen, size_t datalen, bool& isNewAddition) { TNode* current = AddEntryForSomething(key, keylen, isNewAddition); - NodeReleasePayload(current); - if (datalen <= PayloadSize) { - current->PayloadType = DATA_INSIDE; + NodeReleasePayload(current); + if (datalen <= PayloadSize) { + current->PayloadType = DATA_INSIDE; } else if (Flags & CTBF_PREFIX_GROUPED) { - current->PayloadType = DATA_MALLOCED; - current->PayloadAsPtr() = new char[datalen]; - } else { - current->PayloadType = DATA_IN_MEMPOOL; - current->PayloadAsPtr() = (char*) Pool.Allocate(datalen); // XXX: allocate unaligned - } - return current->GetPayload(); + current->PayloadType = DATA_MALLOCED; + current->PayloadAsPtr() = new char[datalen]; + } else { + current->PayloadType = DATA_IN_MEMPOOL; + current->PayloadAsPtr() = (char*) Pool.Allocate(datalen); // XXX: allocate unaligned + } + return current->GetPayload(); } template <class T, class D, class S> @@ -741,19 +741,19 @@ bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::FindLongestPrefixImp template <class T, class D, class S> void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::Clear() { - DestroyNode(Root); + DestroyNode(Root); Pool.Clear(); - NodeAllocator.Reset(new TFixedSizeAllocator(sizeof(TNode) + PayloadSize, TDefaultAllocator::Instance())); - Root = new (*NodeAllocator) TNode; + NodeAllocator.Reset(new TFixedSizeAllocator(sizeof(TNode) + PayloadSize, TDefaultAllocator::Instance())); + Root = new (*NodeAllocator) TNode; EntryCount = 0; NodeCount = 1; } template <class T, class D, class S> size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::Save(IOutputStream& os) const { - const size_t len = NodeMeasureSubtree(Root); - if (len != NodeSaveSubtree(Root, os)) - ythrow yexception() << "something wrong"; + const size_t len = NodeMeasureSubtree(Root); + if (len != NodeSaveSubtree(Root, os)) + ythrow yexception() << "something wrong"; return len; } @@ -781,13 +781,13 @@ template <class T, class D, class S> typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode* TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeForwardAdd( TNode* thiz, const char* label, size_t len, size_t& passed, size_t* nodeCount) { - typename TNode::TArcSet* arcSet = dynamic_cast<typename TNode::TArcSet*>(thiz->Subtree()); + typename TNode::TArcSet* arcSet = dynamic_cast<typename TNode::TArcSet*>(thiz->Subtree()); if (!arcSet) ythrow yexception() << "Bad input order - expected input strings to be prefix-grouped."; - typename TNode::TArcSet::iterator it = arcSet->Find(*label); + typename TNode::TArcSet::iterator it = arcSet->Find(*label); - if (it != arcSet->end()) { + if (it != arcSet->end()) { const char* arcLabel = it->Label.AsCharPtr(); size_t arcLabelLen = it->Label.Length(); @@ -797,8 +797,8 @@ typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode* if (passed < arcLabelLen) { (*nodeCount)++; - TNode* node = new (*NodeAllocator) TNode(); - NodeLinkTo(node, it->Label.SubBlob(passed, arcLabelLen), it->Node); + TNode* node = new (*NodeAllocator) TNode(); + NodeLinkTo(node, it->Label.SubBlob(passed, arcLabelLen), it->Node); it->Node = node; it->Label = it->Label.SubBlob(passed); @@ -811,26 +811,26 @@ typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode* } template <class T, class D, class S> -void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeLinkTo(TNode* thiz, const TBlob& label, TNode* node) { - typename TNode::TArcSet* arcSet = dynamic_cast<typename TNode::TArcSet*>(thiz->Subtree()); - if (!arcSet) +void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeLinkTo(TNode* thiz, const TBlob& label, TNode* node) { + typename TNode::TArcSet* arcSet = dynamic_cast<typename TNode::TArcSet*>(thiz->Subtree()); + if (!arcSet) ythrow yexception() << "Bad input order - expected input strings to be prefix-grouped."; // Buffer the node at the last arc if ((Flags & CTBF_PREFIX_GROUPED) && !arcSet->empty()) - NodeBufferSubtree(arcSet->back().Node); + NodeBufferSubtree(arcSet->back().Node); - arcSet->Add(label, node); + arcSet->Add(label, node); } template <class T, class D, class S> -size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeMeasureSubtree(TNode* thiz) const { +size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeMeasureSubtree(TNode* thiz) const { return (size_t)thiz->Subtree()->Measure(this); } template <class T, class D, class S> ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeSaveSubtree(TNode* thiz, IOutputStream& os) const { - return thiz->Subtree()->Save(this, os); + return thiz->Subtree()->Save(this, os); } template <class T, class D, class S> @@ -839,52 +839,52 @@ ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeSaveSubtreeAndDe } template <class T, class D, class S> -void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeBufferSubtree(TNode* thiz) { - typedef typename TNode::TArcSet TArcSet; - - TArcSet* arcSet = dynamic_cast<TArcSet*>(thiz->Subtree()); - if (!arcSet) +void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeBufferSubtree(TNode* thiz) { + typedef typename TNode::TArcSet TArcSet; + + TArcSet* arcSet = dynamic_cast<TArcSet*>(thiz->Subtree()); + if (!arcSet) return; size_t bufferLength = (size_t)arcSet->Measure(this); - TArrayWithSizeHolder<char> buffer; - buffer.Resize(bufferLength); + TArrayWithSizeHolder<char> buffer; + buffer.Resize(bufferLength); + + TMemoryOutput bufout(buffer.Get(), buffer.Size()); - TMemoryOutput bufout(buffer.Get(), buffer.Size()); - - ui64 written = arcSet->Save(this, bufout); + ui64 written = arcSet->Save(this, bufout); Y_ASSERT(written == bufferLength); - - arcSet->Destroy(this); - arcSet->~TArcSet(); - typename TNode::TBufferedSubtree* bufferedArcSet = new (thiz->Subtree()) typename TNode::TBufferedSubtree; - - bufferedArcSet->Buffer.Swap(buffer); + arcSet->Destroy(this); + arcSet->~TArcSet(); + + typename TNode::TBufferedSubtree* bufferedArcSet = new (thiz->Subtree()) typename TNode::TBufferedSubtree; + + bufferedArcSet->Buffer.Swap(buffer); } template <class T, class D, class S> -size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeMeasureLeafValue(TNode* thiz) const { - if (!thiz->IsFinal()) +size_t TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeMeasureLeafValue(TNode* thiz) const { + if (!thiz->IsFinal()) return 0; - return Packer.SkipLeaf(thiz->GetPayload()); + return Packer.SkipLeaf(thiz->GetPayload()); } template <class T, class D, class S> ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::NodeSaveLeafValue(TNode* thiz, IOutputStream& os) const { - if (!thiz->IsFinal()) - return 0; + if (!thiz->IsFinal()) + return 0; - size_t len = Packer.SkipLeaf(thiz->GetPayload()); - os.Write(thiz->GetPayload(), len); - return len; + size_t len = Packer.SkipLeaf(thiz->GetPayload()); + os.Write(thiz->GetPayload(), len); + return len; } // TCompactTrieBuilder::TCompactTrieBuilderImpl::TNode::TArc template <class T, class D, class S> -TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TArc::TArc(const TBlob& lbl, TNode* nd) +TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TArc::TArc(const TBlob& lbl, TNode* nd) : Label(lbl) , Node(nd) , LeftOffset(0) @@ -896,11 +896,11 @@ ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcMeasure( const TArc* thiz, size_t leftsize, size_t rightsize) const { using namespace NCompactTrie; - size_t coresize = 2 + NodeMeasureLeafValue(thiz->Node); // 2 == (char + flags) - size_t treesize = NodeMeasureSubtree(thiz->Node); + size_t coresize = 2 + NodeMeasureLeafValue(thiz->Node); // 2 == (char + flags) + size_t treesize = NodeMeasureSubtree(thiz->Node); - if (thiz->Label.Length() > 0) - treesize += 2 * (thiz->Label.Length() - 1); + if (thiz->Label.Length() > 0) + treesize += 2 * (thiz->Label.Length() - 1); // Triple measurements are needed because the space needed to store the offset // shall be added to the offset itself. Hence three iterations. @@ -912,8 +912,8 @@ ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcMeasure( rightoffsetsize = rightsize ? MeasureOffset(coresize + treesize + leftsize + leftoffsetsize + rightoffsetsize) : 0; coresize += leftoffsetsize + rightoffsetsize; - thiz->LeftOffset = leftsize ? coresize + treesize : 0; - thiz->RightOffset = rightsize ? coresize + treesize + leftsize : 0; + thiz->LeftOffset = leftsize ? coresize + treesize : 0; + thiz->RightOffset = rightsize ? coresize + treesize + leftsize : 0; return coresize + treesize + leftsize + rightsize; } @@ -922,12 +922,12 @@ template <class T, class D, class S> ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSaveSelf(const TArc* thiz, IOutputStream& os) const { using namespace NCompactTrie; - ui64 written = 0; - - size_t leftoffsetsize = MeasureOffset(thiz->LeftOffset); - size_t rightoffsetsize = MeasureOffset(thiz->RightOffset); + ui64 written = 0; - size_t labelLen = thiz->Label.Length(); + size_t leftoffsetsize = MeasureOffset(thiz->LeftOffset); + size_t rightoffsetsize = MeasureOffset(thiz->RightOffset); + + size_t labelLen = thiz->Label.Length(); for (size_t i = 0; i < labelLen; ++i) { char flags = 0; @@ -938,34 +938,34 @@ ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSaveSelf(const TA } if (i == labelLen-1) { - if (thiz->Node->IsFinal()) + if (thiz->Node->IsFinal()) flags |= MT_FINAL; - if (!thiz->Node->IsLast()) + if (!thiz->Node->IsLast()) flags |= MT_NEXT; } else { flags |= MT_NEXT; } os.Write(&flags, 1); - os.Write(&thiz->Label.AsCharPtr()[i], 1); - written += 2; + os.Write(&thiz->Label.AsCharPtr()[i], 1); + written += 2; if (i == 0) { - written += ArcSaveOffset(thiz->LeftOffset, os); - written += ArcSaveOffset(thiz->RightOffset, os); + written += ArcSaveOffset(thiz->LeftOffset, os); + written += ArcSaveOffset(thiz->RightOffset, os); } } - written += NodeSaveLeafValue(thiz->Node, os); + written += NodeSaveLeafValue(thiz->Node, os); return written; } template <class T, class D, class S> ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSave(const TArc* thiz, IOutputStream& os) const { ui64 written = ArcSaveSelf(thiz, os); - written += NodeSaveSubtree(thiz->Node, os); - return written; + written += NodeSaveSubtree(thiz->Node, os); + return written; } template <class T, class D, class S> diff --git a/library/cpp/containers/comptrie/comptrie_impl.cpp b/library/cpp/containers/comptrie/comptrie_impl.cpp index 309176e0ec..a116ab6d1e 100644 --- a/library/cpp/containers/comptrie/comptrie_impl.cpp +++ b/library/cpp/containers/comptrie/comptrie_impl.cpp @@ -1,6 +1,6 @@ #include "comptrie_impl.h" -#include <util/system/rusage.h> +#include <util/system/rusage.h> #include <util/stream/output.h> // Unpack the leaf value. The algorithm can store up to 8 full bytes in leafs. diff --git a/library/cpp/containers/comptrie/comptrie_impl.h b/library/cpp/containers/comptrie/comptrie_impl.h index 2cbae822bf..f41c38311a 100644 --- a/library/cpp/containers/comptrie/comptrie_impl.h +++ b/library/cpp/containers/comptrie/comptrie_impl.h @@ -1,6 +1,6 @@ #pragma once -#include <util/stream/output.h> +#include <util/stream/output.h> #ifndef COMPTRIE_DATA_CHECK #define COMPTRIE_DATA_CHECK 1 @@ -11,10 +11,10 @@ namespace NCompactTrie { const char MT_FINAL = '\x80'; const char MT_NEXT = '\x40'; - const char MT_SIZEMASK = '\x07'; + const char MT_SIZEMASK = '\x07'; const size_t MT_LEFTSHIFT = 3; - const size_t MT_RIGHTSHIFT = 0; - + const size_t MT_RIGHTSHIFT = 0; + Y_FORCE_INLINE size_t UnpackOffset(const char* p, size_t len); size_t MeasureOffset(size_t offset); size_t PackOffset(char* buffer, size_t offset); @@ -52,20 +52,20 @@ namespace NCompactTrie { void ShowProgress(size_t n); // just print dots } -namespace NCompTriePrivate { - template <typename TChar> - struct TStringForChar { - }; - - template <> - struct TStringForChar<char> { +namespace NCompTriePrivate { + template <typename TChar> + struct TStringForChar { + }; + + template <> + struct TStringForChar<char> { typedef TString TResult; - }; - - template <> - struct TStringForChar<wchar16> { + }; + + template <> + struct TStringForChar<wchar16> { typedef TUtf16String TResult; - }; + }; template <> struct TStringForChar<wchar32> { @@ -73,7 +73,7 @@ namespace NCompTriePrivate { }; } - + namespace NCompTriePrivate { struct TCmp { template <class T> @@ -88,18 +88,18 @@ namespace NCompTriePrivate { }; } -namespace NCompactTrie { +namespace NCompactTrie { static inline ui64 ArcSaveOffset(size_t offset, IOutputStream& os) { - using namespace NCompactTrie; - - if (!offset) - return 0; - - char buf[16]; - size_t len = PackOffset(buf, offset); - os.Write(buf, len); - return len; - } + using namespace NCompactTrie; + + if (!offset) + return 0; + + char buf[16]; + size_t len = PackOffset(buf, offset); + os.Write(buf, len); + return len; + } // Unpack the offset to the next node. The encoding scheme can store offsets // up to 7 bytes; whether they fit into size_t is another issue. @@ -218,4 +218,4 @@ namespace NCompactTrie { return false; } -} +} diff --git a/library/cpp/containers/comptrie/comptrie_trie.h b/library/cpp/containers/comptrie/comptrie_trie.h index 534f7d424f..40ec1e52b3 100644 --- a/library/cpp/containers/comptrie/comptrie_trie.h +++ b/library/cpp/containers/comptrie/comptrie_trie.h @@ -14,9 +14,9 @@ #include <util/stream/input.h> #include <utility> -template <class T, class D, class S> -class TCompactTrieBuilder; - +template <class T, class D, class S> +class TCompactTrieBuilder; + namespace NCompactTrie { template <class TTrie> class TFirstSymbolIterator; @@ -38,17 +38,17 @@ public: typedef typename TCompactTrieKeySelector<TSymbol>::TKey TKey; typedef typename TCompactTrieKeySelector<TSymbol>::TKeyBuf TKeyBuf; - + typedef std::pair<TKey, TData> TValueType; typedef std::pair<size_t, TData> TPhraseMatch; typedef TVector<TPhraseMatch> TPhraseMatchVector; - typedef TCompactTrieBuilder<T, D, S> TBuilder; - + typedef TCompactTrieBuilder<T, D, S> TBuilder; + protected: TBlob DataHolder; const char* EmptyValue = nullptr; - TPacker Packer; + TPacker Packer; NCompactTrie::TPackerLeafSkipper<TPacker> Skipper = &Packer; // This should be true for every constructor. public: @@ -74,7 +74,7 @@ public: return !IsEmpty(); } - void Init(const char* d, size_t len, TPacker packer = TPacker()); + void Init(const char* d, size_t len, TPacker packer = TPacker()); void Init(const TBlob& data, TPacker packer = TPacker()); bool IsInitialized() const; @@ -83,17 +83,17 @@ public: bool Find(const TSymbol* key, size_t keylen, TData* value = nullptr) const; bool Find(const TKeyBuf& key, TData* value = nullptr) const { return Find(key.data(), key.size(), value); - } - - TData Get(const TSymbol* key, size_t keylen) const { - TData value; - if (!Find(key, keylen, &value)) - ythrow yexception() << "key " << TKey(key, keylen).Quote() << " not found in trie"; - return value; - } + } + + TData Get(const TSymbol* key, size_t keylen) const { + TData value; + if (!Find(key, keylen, &value)) + ythrow yexception() << "key " << TKey(key, keylen).Quote() << " not found in trie"; + return value; + } TData Get(const TKeyBuf& key) const { return Get(key.data(), key.size()); - } + } TData GetDefault(const TKeyBuf& key, const TData& def) const { TData value; if (!Find(key.data(), key.size(), &value)) @@ -101,7 +101,7 @@ public: else return value; } - + const TBlob& Data() const { return DataHolder; }; @@ -121,11 +121,11 @@ public: void FindPhrases(const TSymbol* key, size_t keylen, TPhraseMatchVector& matches, TSymbol separator = TSymbol(' ')) const; void FindPhrases(const TKeyBuf& key, TPhraseMatchVector& matches, TSymbol separator = TSymbol(' ')) const { return FindPhrases(key.data(), key.size(), matches, separator); - } + } bool FindLongestPrefix(const TSymbol* key, size_t keylen, size_t* prefixLen, TData* value = nullptr, bool* hasNext = nullptr) const; bool FindLongestPrefix(const TKeyBuf& key, size_t* prefixLen, TData* value = nullptr, bool* hasNext = nullptr) const { return FindLongestPrefix(key.data(), key.size(), prefixLen, value, hasNext); - } + } // Return trie, containing all tails for the given key inline TCompactTrie<T, D, S> FindTails(const TSymbol* key, size_t keylen) const; @@ -154,7 +154,7 @@ public: bool IsEmpty() const { return !Impl; } // Almost no other method can be called. - + bool operator==(const TConstIterator& other) const; bool operator!=(const TConstIterator& other) const; TConstIterator& operator++(); @@ -163,12 +163,12 @@ public: TConstIterator operator--(int /*unused*/); TValueType operator*(); - TKey GetKey() const; + TKey GetKey() const; size_t GetKeySize() const; - TData GetValue() const; + TData GetValue() const; void GetValue(TData& data) const; - const char* GetValuePtr() const; - + const char* GetValuePtr() const; + private: TPacker Packer; TCopyPtr<TOpaqueTrieIterator> Impl; @@ -186,7 +186,7 @@ public: TConstIterator UpperBound(const TKeyBuf& key) const; void Print(IOutputStream& os); - + size_t Size() const; friend class NCompactTrie::TFirstSymbolIterator<TCompactTrie>; @@ -224,7 +224,7 @@ public: template <class T, class D, class S> TCompactTrie<T, D, S>::TCompactTrie(const TBlob& data, TPacker packer) : DataHolder(data) - , Packer(packer) + , Packer(packer) { Init(data, packer); } @@ -246,7 +246,7 @@ template <class T, class D, class S> TCompactTrie<T, D, S>::TCompactTrie(const TBlob& data, const char* emptyValue, TPacker packer) : DataHolder(data) , EmptyValue(emptyValue) - , Packer(packer) + , Packer(packer) { } @@ -296,7 +296,7 @@ void TCompactTrie<T, D, S>::Init(const TBlob& data, TPacker packer) { using namespace NCompactTrie; DataHolder = data; - Packer = packer; + Packer = packer; const char* datapos = DataHolder.AsCharPtr(); size_t len = DataHolder.Length(); @@ -454,11 +454,11 @@ typename TCompactTrie<T, D, S>::TConstIterator TCompactTrie<T, D, S>::UpperBound template <class T, class D, class S> void TCompactTrie<T, D, S>::Print(IOutputStream& os) { typedef typename ::TCompactTrieKeySelector<T>::TKeyBuf TSBuffer; - for (TConstIterator it = Begin(); it != End(); ++it) { + for (TConstIterator it = Begin(); it != End(); ++it) { os << TSBuffer((*it).first.data(), (*it).first.size()) << "\t" << (*it).second << Endl; - } -} - + } +} + template <class T, class D, class S> bool TCompactTrie<T, D, S>::FindLongestPrefix(const TSymbol* key, size_t keylen, size_t* prefixLen, TData* value, bool* hasNext) const { const char* valuepos = nullptr; @@ -654,10 +654,10 @@ typename TCompactTrie<T, D, S>::TData TCompactTrie<T, D, S>::TConstIterator::Get template <class T, class D, class S> void TCompactTrie<T, D, S>::TConstIterator::GetValue(typename TCompactTrie<T, D, S>::TData& data) const { - const char* ptr = GetValuePtr(); + const char* ptr = GetValuePtr(); if (ptr) { Packer.UnpackLeaf(ptr, data); - } else { + } else { data = typename TCompactTrie<T, D, S>::TData(); - } -} + } +} diff --git a/library/cpp/containers/comptrie/comptrie_ut.cpp b/library/cpp/containers/comptrie/comptrie_ut.cpp index 2db5143c60..74bee09b5d 100644 --- a/library/cpp/containers/comptrie/comptrie_ut.cpp +++ b/library/cpp/containers/comptrie/comptrie_ut.cpp @@ -12,8 +12,8 @@ #include <util/generic/ptr.h> #include <util/generic/ylimits.h> -#include <util/folder/dirut.h> - +#include <util/folder/dirut.h> + #include <util/random/random.h> #include <util/random/fast.h> @@ -369,31 +369,31 @@ void TCompactTrieTest::CheckData(const char* data, size_t datalen) { size_t prefixLen = 0; typename TCompactTrie<T>::TKey key = MakeWideKey<T>(i, len); - UNIT_ASSERT(trie.Find(key, &value)); + UNIT_ASSERT(trie.Find(key, &value)); UNIT_ASSERT_EQUAL(len * 2, value); - UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, &value)); + UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, &value)); UNIT_ASSERT_EQUAL(len, prefixLen); UNIT_ASSERT_EQUAL(len * 2, value); TString badkey("bb"); badkey += i; key = MakeWideKey<T>(badkey); - UNIT_ASSERT(!trie.Find(key)); + UNIT_ASSERT(!trie.Find(key)); value = 123; - UNIT_ASSERT(!trie.Find(key, &value)); + UNIT_ASSERT(!trie.Find(key, &value)); UNIT_ASSERT_EQUAL(123, value); - UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, &value)); + UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, &value)); UNIT_ASSERT_EQUAL(1, prefixLen); UNIT_ASSERT_EQUAL(2, value); badkey = i; badkey += "x"; key = MakeWideKey<T>(badkey); - UNIT_ASSERT(!trie.Find(key)); + UNIT_ASSERT(!trie.Find(key)); value = 1234; - UNIT_ASSERT(!trie.Find(key, &value)); + UNIT_ASSERT(!trie.Find(key, &value)); UNIT_ASSERT_EQUAL(1234, value); - UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, &value)); + UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, &value)); UNIT_ASSERT_EQUAL(len, prefixLen); UNIT_ASSERT_EQUAL(len * 2, value); UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, nullptr)); @@ -410,12 +410,12 @@ void TCompactTrieTest::CheckData(const char* data, size_t datalen) { testkey = "fbbax"; key = MakeWideKey<T>(testkey); - UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, &value)); + UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, &value)); UNIT_ASSERT_EQUAL(prefixLen, 3); UNIT_ASSERT_EQUAL(6, value); value = 12345678; - UNIT_ASSERT(!trie.Find(key, &value)); + UNIT_ASSERT(!trie.Find(key, &value)); UNIT_ASSERT_EQUAL(12345678, value); //Failed Find() should not change value } @@ -689,7 +689,7 @@ void TCompactTrieTest::TestRandom(const size_t n, const size_t maxKeySize) { TCompactTrie<char, typename T::TData, T> trieMin(buftmp.Buffer().Data(), buftmp.Buffer().Size()); TCompactTrieBuilder<char, typename T::TData, T> prefixGroupedBuilder(CTBF_PREFIX_GROUPED); - + for (typename TKeys::const_iterator i = keys.begin(), mi = keys.end(); i != mi; ++i) { UNIT_ASSERT(!prefixGroupedBuilder.Find(i->first.c_str(), i->first.size(), &dummy)); UNIT_ASSERT(trie.Find(i->first.c_str(), i->first.size(), &dummy)); @@ -698,7 +698,7 @@ void TCompactTrieTest::TestRandom(const size_t n, const size_t maxKeySize) { UNIT_ASSERT(trieMin.Find(i->first.c_str(), i->first.size(), &dummy)); UNIT_ASSERT(dummy == i->second); } - + prefixGroupedBuilder.Add(i->first.c_str(), i->first.size(), dummy); UNIT_ASSERT(prefixGroupedBuilder.Find(i->first.c_str(), i->first.size(), &dummy)); @@ -712,10 +712,10 @@ void TCompactTrieTest::TestRandom(const size_t n, const size_t maxKeySize) { } } } - + TBufferStream prefixGroupedBuffer; prefixGroupedBuilder.Save(prefixGroupedBuffer); - + UNIT_ASSERT_VALUES_EQUAL(stream.Buffer().Size(), prefixGroupedBuffer.Buffer().Size()); UNIT_ASSERT(0 == memcmp(stream.Buffer().Data(), prefixGroupedBuffer.Buffer().Data(), stream.Buffer().Size())); } @@ -776,9 +776,9 @@ void TCompactTrieTest::TestFindTailsImpl(const TString& prefix) { } void TCompactTrieTest::TestPrefixGrouped() { - TBuffer b1b; + TBuffer b1b; TCompactTrieBuilder<char, ui32> b1(CTBF_PREFIX_GROUPED); - const char* data[] = { + const char* data[] = { "Kazan", "Moscow", "Monino", @@ -787,8 +787,8 @@ void TCompactTrieTest::TestPrefixGrouped() { "Fryazino", "Fryazevo", "Tumen", - }; - + }; + for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { ui32 val = strlen(data[i]) + 1; b1.Add(data[i], strlen(data[i]), val); @@ -802,24 +802,24 @@ void TCompactTrieTest::TestPrefixGrouped() { UNIT_ASSERT(!b1.Find(data[j], strlen(data[j]), &found)); } } - } - - { - TBufferOutput b1bo(b1b); - b1.Save(b1bo); - } - - TCompactTrie<char, ui32> t1(TBlob::FromBuffer(b1b)); - - //t1.Print(Cerr); - + } + + { + TBufferOutput b1bo(b1b); + b1.Save(b1bo); + } + + TCompactTrie<char, ui32> t1(TBlob::FromBuffer(b1b)); + + //t1.Print(Cerr); + for (auto& i : data) { - ui32 v; + ui32 v; UNIT_ASSERT(t1.Find(i, strlen(i), &v)); UNIT_ASSERT_VALUES_EQUAL(strlen(i) + 1, v); - } -} - + } +} + void TCompactTrieTest::CrashTestPrefixGrouped() { TCompactTrieBuilder<char, ui32> builder(CTBF_PREFIX_GROUPED); const char* data[] = { @@ -842,33 +842,33 @@ void TCompactTrieTest::CrashTestPrefixGrouped() { } void TCompactTrieTest::TestMergeFromFile() { - { - TCompactTrieBuilder<> b; - b.Add("yandex", 12); - b.Add("google", 13); - b.Add("mail", 14); + { + TCompactTrieBuilder<> b; + b.Add("yandex", 12); + b.Add("google", 13); + b.Add("mail", 14); TUnbufferedFileOutput out(GetSystemTempDir() + "/TCompactTrieTest-TestMerge-ru"); - b.Save(out); - } - - { - TCompactTrieBuilder<> b; - b.Add("yandex", 112); - b.Add("google", 113); - b.Add("yahoo", 114); + b.Save(out); + } + + { + TCompactTrieBuilder<> b; + b.Add("yandex", 112); + b.Add("google", 113); + b.Add("yahoo", 114); TUnbufferedFileOutput out(GetSystemTempDir() + "/TCompactTrieTest-TestMerge-com"); - b.Save(out); - } - - { - TCompactTrieBuilder<> b; + b.Save(out); + } + + { + TCompactTrieBuilder<> b; UNIT_ASSERT(b.AddSubtreeInFile("com.", GetSystemTempDir() + "/TCompactTrieTest-TestMerge-com")); UNIT_ASSERT(b.Add("org.kernel", 22)); UNIT_ASSERT(b.AddSubtreeInFile("ru.", GetSystemTempDir() + "/TCompactTrieTest-TestMerge-ru")); TUnbufferedFileOutput out(GetSystemTempDir() + "/TCompactTrieTest-TestMerge-res"); - b.Save(out); - } - + b.Save(out); + } + TCompactTrie<> trie(TBlob::FromFileSingleThreaded(GetSystemTempDir() + "/TCompactTrieTest-TestMerge-res")); UNIT_ASSERT_VALUES_EQUAL(12u, trie.Get("ru.yandex")); UNIT_ASSERT_VALUES_EQUAL(13u, trie.Get("ru.google")); @@ -877,12 +877,12 @@ void TCompactTrieTest::TestMergeFromFile() { UNIT_ASSERT_VALUES_EQUAL(112u, trie.Get("com.yandex")); UNIT_ASSERT_VALUES_EQUAL(113u, trie.Get("com.google")); UNIT_ASSERT_VALUES_EQUAL(114u, trie.Get("com.yahoo")); - + unlink((GetSystemTempDir() + "/TCompactTrieTest-TestMerge-res").data()); unlink((GetSystemTempDir() + "/TCompactTrieTest-TestMerge-com").data()); unlink((GetSystemTempDir() + "/TCompactTrieTest-TestMerge-ru").data()); -} - +} + void TCompactTrieTest::TestMergeFromBuffer() { TArrayWithSizeHolder<char> buffer1; { diff --git a/library/cpp/containers/comptrie/make_fast_layout.h b/library/cpp/containers/comptrie/make_fast_layout.h index 7325c284c6..b8fab5d65b 100644 --- a/library/cpp/containers/comptrie/make_fast_layout.h +++ b/library/cpp/containers/comptrie/make_fast_layout.h @@ -4,7 +4,7 @@ #include <cstddef> class IOutputStream; - + namespace NCompactTrie { // Return value: size of the resulting trie. size_t RawCompactTrieFastLayoutImpl(IOutputStream& os, const NCompactTrie::TOpaqueTrie& trie, bool verbose); diff --git a/library/cpp/containers/comptrie/minimize.h b/library/cpp/containers/comptrie/minimize.h index 354c50334e..baaa431d04 100644 --- a/library/cpp/containers/comptrie/minimize.h +++ b/library/cpp/containers/comptrie/minimize.h @@ -4,10 +4,10 @@ #include <cstddef> class IOutputStream; - + namespace NCompactTrie { size_t MeasureOffset(size_t offset); - + enum EMinimizeMode { MM_DEFAULT, // alollocate new memory for minimized tree MM_NOALLOC, // minimize tree in the same buffer diff --git a/library/cpp/containers/comptrie/opaque_trie_iterator.cpp b/library/cpp/containers/comptrie/opaque_trie_iterator.cpp index 3b2ac3d157..5fd3914be6 100644 --- a/library/cpp/containers/comptrie/opaque_trie_iterator.cpp +++ b/library/cpp/containers/comptrie/opaque_trie_iterator.cpp @@ -13,7 +13,7 @@ namespace NCompactTrie { if (!AtEmptyValue && !atend) Forward(); } - + bool TOpaqueTrieIterator::operator==(const TOpaqueTrieIterator& rhs) const { return (Trie == rhs.Trie && Forks == rhs.Forks && @@ -52,7 +52,7 @@ namespace NCompactTrie { return false; topFork = &Forks.Top(); } - } + } Y_ASSERT(!Forks.Empty()); while (Forks.Top().CurrentDirection != D_FINAL && !HasMaxKeyLength()) { @@ -228,4 +228,4 @@ namespace NCompactTrie { return Node.GetLeafOffset(); } -} +} diff --git a/library/cpp/containers/comptrie/opaque_trie_iterator.h b/library/cpp/containers/comptrie/opaque_trie_iterator.h index e2ed30a3cd..195da3c191 100644 --- a/library/cpp/containers/comptrie/opaque_trie_iterator.h +++ b/library/cpp/containers/comptrie/opaque_trie_iterator.h @@ -10,7 +10,7 @@ namespace NCompactTrie { class ILeafSkipper; - + class TFork { // Auxiliary class for a branching point in the iterator public: TNode Node; diff --git a/library/cpp/containers/comptrie/ya.make b/library/cpp/containers/comptrie/ya.make index fee6680aba..81352da4b2 100644 --- a/library/cpp/containers/comptrie/ya.make +++ b/library/cpp/containers/comptrie/ya.make @@ -1,8 +1,8 @@ LIBRARY() - + OWNER(velavokr) -SRCS( +SRCS( array_with_size.h chunked_helpers_trie.h comptrie.h @@ -23,8 +23,8 @@ SRCS( search_iterator.cpp write_trie_backwards.cpp writeable_node.cpp -) - +) + PEERDIR( library/cpp/packers library/cpp/containers/compact_vector @@ -32,4 +32,4 @@ PEERDIR( util/draft ) -END() +END() diff --git a/library/cpp/coroutine/engine/impl.cpp b/library/cpp/coroutine/engine/impl.cpp index 95e598701c..7ae6f74051 100644 --- a/library/cpp/coroutine/engine/impl.cpp +++ b/library/cpp/coroutine/engine/impl.cpp @@ -7,7 +7,7 @@ #include <util/thread/singleton.h> #include <util/stream/format.h> #include <util/stream/output.h> -#include <util/system/yassert.h> +#include <util/system/yassert.h> TCont::TJoinWait::TJoinWait(TCont& c) noexcept diff --git a/library/cpp/coroutine/engine/impl.h b/library/cpp/coroutine/engine/impl.h index d81e81fdf2..283a96ecf1 100644 --- a/library/cpp/coroutine/engine/impl.h +++ b/library/cpp/coroutine/engine/impl.h @@ -143,8 +143,8 @@ public: virtual void Execute() = 0; }; -/// Central coroutine class. -/// Note, coroutines are single-threaded, and all methods must be called from the single thread +/// Central coroutine class. +/// Note, coroutines are single-threaded, and all methods must be called from the single thread class TContExecutor { friend class TCont; using TContList = TIntrusiveList<TCont>; diff --git a/library/cpp/coroutine/engine/trampoline.cpp b/library/cpp/coroutine/engine/trampoline.cpp index a73513ded8..10ea69ddc3 100644 --- a/library/cpp/coroutine/engine/trampoline.cpp +++ b/library/cpp/coroutine/engine/trampoline.cpp @@ -6,7 +6,7 @@ #include <util/system/info.h> #include <util/system/protect.h> #include <util/system/valgrind.h> -#include <util/system/yassert.h> +#include <util/system/yassert.h> #include <cstdlib> #include <util/stream/format.h> diff --git a/library/cpp/coroutine/listener/listen.h b/library/cpp/coroutine/listener/listen.h index 5b39d75c2c..3a89cd3ecc 100644 --- a/library/cpp/coroutine/listener/listen.h +++ b/library/cpp/coroutine/listener/listen.h @@ -74,7 +74,7 @@ public: const NAddr::IRemoteAddr* Remote; const NAddr::IRemoteAddr* Local; }; - + virtual void OnAccept(const TAccept&) { } diff --git a/library/cpp/deprecated/enum_codegen/enum_codegen.h b/library/cpp/deprecated/enum_codegen/enum_codegen.h index 49f723354d..dfb04ecac2 100644 --- a/library/cpp/deprecated/enum_codegen/enum_codegen.h +++ b/library/cpp/deprecated/enum_codegen/enum_codegen.h @@ -1,10 +1,10 @@ -#pragma once - -/// see enum_codegen_ut.cpp for examples - -#define ENUM_VALUE_GEN(name, value, ...) name = value, -#define ENUM_VALUE_GEN_NO_VALUE(name, ...) name, - +#pragma once + +/// see enum_codegen_ut.cpp for examples + +#define ENUM_VALUE_GEN(name, value, ...) name = value, +#define ENUM_VALUE_GEN_NO_VALUE(name, ...) name, + #define ENUM_TO_STRING_IMPL_ITEM(name, ...) \ case name: \ return #name; @@ -12,7 +12,7 @@ case name: \ os << #name; \ break; - + #define ENUM_TO_STRING(type, MAP) \ static inline const char* ToCString(type value) { \ switch (value) { \ diff --git a/library/cpp/deprecated/enum_codegen/enum_codegen_ut.cpp b/library/cpp/deprecated/enum_codegen/enum_codegen_ut.cpp index 5a25265f2e..f8f1c9b6df 100644 --- a/library/cpp/deprecated/enum_codegen/enum_codegen_ut.cpp +++ b/library/cpp/deprecated/enum_codegen/enum_codegen_ut.cpp @@ -1,40 +1,40 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "enum_codegen.h" - + +#include "enum_codegen.h" + #include <util/string/builder.h> -#define COLOR_MAP(XX) \ +#define COLOR_MAP(XX) \ XX(RED) \ XX(GREEN) \ - XX(BLUE) - -enum EColor { - COLOR_MAP(ENUM_VALUE_GEN_NO_VALUE) -}; - -ENUM_TO_STRING(EColor, COLOR_MAP) - -#define MULTIPLIER_MAP(XX) \ + XX(BLUE) + +enum EColor { + COLOR_MAP(ENUM_VALUE_GEN_NO_VALUE) +}; + +ENUM_TO_STRING(EColor, COLOR_MAP) + +#define MULTIPLIER_MAP(XX) \ XX(GB, 9) \ XX(MB, 6) \ - XX(KB, 3) - -enum EMultiplier { - MULTIPLIER_MAP(ENUM_VALUE_GEN) -}; - -ENUM_TO_STRING(EMultiplier, MULTIPLIER_MAP) - + XX(KB, 3) + +enum EMultiplier { + MULTIPLIER_MAP(ENUM_VALUE_GEN) +}; + +ENUM_TO_STRING(EMultiplier, MULTIPLIER_MAP) + Y_UNIT_TEST_SUITE(EnumCodegen) { Y_UNIT_TEST(GenWithValue) { - UNIT_ASSERT_VALUES_EQUAL(6, MB); - } - + UNIT_ASSERT_VALUES_EQUAL(6, MB); + } + Y_UNIT_TEST(ToCString) { - UNIT_ASSERT_VALUES_EQUAL("RED", ToCString(RED)); - UNIT_ASSERT_VALUES_EQUAL("BLUE", ToCString(BLUE)); + UNIT_ASSERT_VALUES_EQUAL("RED", ToCString(RED)); + UNIT_ASSERT_VALUES_EQUAL("BLUE", ToCString(BLUE)); UNIT_ASSERT_VALUES_EQUAL("GREEN", (TStringBuilder() << GREEN)); UNIT_ASSERT_VALUES_EQUAL("GB", ToCString(GB)); - } -} + } +} diff --git a/library/cpp/deprecated/mapped_file/mapped_file.cpp b/library/cpp/deprecated/mapped_file/mapped_file.cpp index 905b90eaa1..b0e4511299 100644 --- a/library/cpp/deprecated/mapped_file/mapped_file.cpp +++ b/library/cpp/deprecated/mapped_file/mapped_file.cpp @@ -1,5 +1,5 @@ #include "mapped_file.h" - + #include <util/generic/yexception.h> #include <util/system/defaults.h> #include <util/system/hi_lo.h> diff --git a/library/cpp/digest/md5/md5_medium_ut.cpp b/library/cpp/digest/md5/md5_medium_ut.cpp index 34131aeb48..a940c5cb66 100644 --- a/library/cpp/digest/md5/md5_medium_ut.cpp +++ b/library/cpp/digest/md5/md5_medium_ut.cpp @@ -1,7 +1,7 @@ #include "md5.h" #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TMD5MediumTest) { Y_UNIT_TEST(TestOverflow) { if (sizeof(size_t) > sizeof(unsigned int)) { diff --git a/library/cpp/digest/md5/md5_ut.cpp b/library/cpp/digest/md5/md5_ut.cpp index d2a67eb97d..1c3e4ad0a9 100644 --- a/library/cpp/digest/md5/md5_ut.cpp +++ b/library/cpp/digest/md5/md5_ut.cpp @@ -1,7 +1,7 @@ #include "md5.h" #include <library/cpp/testing/unittest/registar.h> - + #include <util/system/fs.h> #include <util/stream/file.h> @@ -40,7 +40,7 @@ Y_UNIT_TEST_SUITE(TMD5Test) { TString memoryHash = MD5::Data((const unsigned char*)s.data(), s.size(), memBuf); UNIT_ASSERT_NO_DIFF(fileHash, memoryHash); - + fileHash = MD5::File(tmpFile); UNIT_ASSERT_NO_DIFF(fileHash, memoryHash); diff --git a/library/cpp/execprofile/profile.cpp b/library/cpp/execprofile/profile.cpp index 883054f869..d05de20203 100644 --- a/library/cpp/execprofile/profile.cpp +++ b/library/cpp/execprofile/profile.cpp @@ -1,5 +1,5 @@ -#include <util/system/defaults.h> - +#include <util/system/defaults.h> + #include "profile.h" #if defined(_unix_) && !defined(_bionic_) && !defined(_cygwin_) @@ -293,7 +293,7 @@ private: }; void TSampleAnalyser::Analyze(FILE* out) const { - fprintf(out, "samples: %" PRIu64 " unique: %" PRIu64 " dropped: %" PRIu64 " searchskips: %" PRIu64 "\n", + fprintf(out, "samples: %" PRIu64 " unique: %" PRIu64 " dropped: %" PRIu64 " searchskips: %" PRIu64 "\n", (ui64)Stats.SavedSamples, (ui64)Samples.size(), (ui64)Stats.DroppedSamples, (ui64)Stats.SearchSkipCount); @@ -306,13 +306,13 @@ void TSampleAnalyser::Analyze(FILE* out) const { // dumping the samples if (PutTimestamps && (i % 1000 == 0)) { ui64 tm = GetCycleCount(); - fprintf(out, "TM: %" PRIu64 "\n", tm); + fprintf(out, "TM: %" PRIu64 "\n", tm); } Dl_info addrInfo; if (dladdr(Samples[i].first, &addrInfo)) { if (addrInfo.dli_fbase != prevModBase || addrInfo.dli_saddr != prevFunc) { - fprintf(out, "Func\t%" PRISZT "\t%p\t%p\t%s\t%s\n", + fprintf(out, "Func\t%" PRISZT "\t%p\t%p\t%s\t%s\n", funcCnt, addrInfo.dli_fbase, addrInfo.dli_saddr, @@ -325,7 +325,7 @@ void TSampleAnalyser::Analyze(FILE* out) const { } else { fprintf(out, "[dladdr failed]\n"); } - fprintf(out, "%" PRISZT "\t%p\t%lu\n", i, Samples[i].first, Samples[i].second); + fprintf(out, "%" PRISZT "\t%p\t%lu\n", i, Samples[i].first, Samples[i].second); } } diff --git a/library/cpp/getopt/last_getopt_demo/demo.cpp b/library/cpp/getopt/last_getopt_demo/demo.cpp index cb31dba733..79426a9cc9 100644 --- a/library/cpp/getopt/last_getopt_demo/demo.cpp +++ b/library/cpp/getopt/last_getopt_demo/demo.cpp @@ -3,7 +3,7 @@ #include <library/cpp/colorizer/colors.h> // For the sake of this example, let's implement Wget - + Y_COMPLETER(HeaderCompleter) { AddCompletion("Host"); AddCompletion("Referer"); @@ -31,8 +31,8 @@ Y_COMPLETER(HeaderCompleter) { AddCompletion("Content-MD5"); AddCompletion("Content-Range"); } -} - +} + class TMain: public TMainClassArgs { bool Background_; size_t Timeout_; @@ -42,7 +42,7 @@ class TMain: public TMainClassArgs { TMaybe<TString> PostData_; TMaybe<TString> PostFile_; TVector<TString> Headers_; - + protected: void RegisterOptions(NLastGetopt::TOpts& opts) override { // Brief description for the whole program, will appear in the beginning of a help message. @@ -75,7 +75,7 @@ protected: .CompletionArgHelp("timeout (ms)") .StoreResult(&Timeout_) .Completer(NLastGetopt::NComp::Choice({{"1000"}, {"5000"}, {"10000"}, {"60000"}})); - + opts.AddLongOption("method") .RequiredArgument("http-method") .Help("specify HTTP method") @@ -100,7 +100,7 @@ protected: .CompletionHelp("set custom user agent for each HTTP request") .CompletionArgHelp("user agent string") .StoreResult(&UserAgent_); - + opts.AddLongOption("post-data") .RequiredArgument("string") .Help("use POST method and send the specified data in the request body (cannot be used with --post-file)") @@ -121,10 +121,10 @@ protected: ImplicitMethod_ = "POST"; }) .Completer(NLastGetopt::NComp::File()); - + // These two options can't be together. opts.MutuallyExclusive("post-file", "post-data"); - + opts.AddLongOption("header") .RequiredArgument("header-line") .Help("send `header-line` along with the rest of the headers in each HTTP request") @@ -133,9 +133,9 @@ protected: .AppendTo(&Headers_) .AllowMultipleCompletion() .Completer(NLastGetopt::NComp::LaunchSelf(HeaderCompleter)); - + // Setting up free arguments. - + // We are going to have one mandatory argument and unlimited number of optional arguments. opts.SetFreeArgsMin(1); opts.SetFreeArgsMax(NLastGetopt::TOpts::UNLIMITED_ARGS); @@ -239,4 +239,4 @@ protected: int main(int argc, const char** argv) { NLastGetopt::NComp::TCustomCompleter::FireCustomCompleter(argc, argv); TMain().Run(argc, argv); -} +} diff --git a/library/cpp/getopt/last_getopt_demo/ya.make b/library/cpp/getopt/last_getopt_demo/ya.make index 275f983051..53f1cfc122 100644 --- a/library/cpp/getopt/last_getopt_demo/ya.make +++ b/library/cpp/getopt/last_getopt_demo/ya.make @@ -1,13 +1,13 @@ -PROGRAM(last_getopt_demo) - +PROGRAM(last_getopt_demo) + OWNER(amatanhead) PEERDIR( library/cpp/getopt ) -SRCS( - demo.cpp -) - -END() +SRCS( + demo.cpp +) + +END() diff --git a/library/cpp/getopt/small/last_getopt.cpp b/library/cpp/getopt/small/last_getopt.cpp index d76420629d..30669b2c5a 100644 --- a/library/cpp/getopt/small/last_getopt.cpp +++ b/library/cpp/getopt/small/last_getopt.cpp @@ -6,4 +6,4 @@ namespace NLastGetopt { exit(0); } -} +} diff --git a/library/cpp/getopt/small/last_getopt.h b/library/cpp/getopt/small/last_getopt.h index 4377591fa8..07687bc914 100644 --- a/library/cpp/getopt/small/last_getopt.h +++ b/library/cpp/getopt/small/last_getopt.h @@ -1,21 +1,21 @@ #pragma once - + #include "last_getopt_opts.h" #include "last_getopt_easy_setup.h" #include "last_getopt_parse_result.h" #include <util/generic/function.h> #include <util/string/split.h> - + /// see some documentation in /// https://wiki.yandex-team.ru/development/poisk/arcadia/util/lastgetopt/ /// https://wiki.yandex-team.ru/development/poisk/arcadia/library/getopt/ /// see examples in library/cpp/getopt/last_getopt_demo - + //TODO: in most cases this include is unnecessary, but needed THandlerFunctor1<TpFunc, TpArg>::HandleOpt #include "last_getopt_parser.h" -namespace NLastGetopt { +namespace NLastGetopt { /// Handler to split option value by delimiter into a target container and allow ranges. template <class Container> struct TOptRangeSplitHandler: public IOptHandler { @@ -35,21 +35,21 @@ namespace NLastGetopt { if (curval.IsInited()) { StringSplitter(curval).Split(ElementsDelim).Consume([&](const TStringBuf& val) { TStringBuf mutableValue = val; - + TValue first = NPrivate::OptFromString<TValue>(mutableValue.NextTok(RangesDelim), parser->CurOpt()); TValue last = mutableValue ? NPrivate::OptFromString<TValue>(mutableValue, parser->CurOpt()) : first; - + if (last < first) { throw TUsageException() << "failed to parse opt " << NPrivate::OptToString(parser->CurOpt()) << " value " << TString(val).Quote() << ": the second argument is less than the first one"; } - + for (++last; first < last; ++first) { Target->insert(Target->end(), first); } }); } } - + private: TContainer* Target; char ElementsDelim; @@ -126,7 +126,7 @@ namespace NLastGetopt { << " value " << TString(curval).Quote() << ": " << CurrentExceptionMessage(); } } - - } + + } } diff --git a/library/cpp/getopt/small/last_getopt_support.h b/library/cpp/getopt/small/last_getopt_support.h index 3ea05733ff..17bed3e614 100644 --- a/library/cpp/getopt/small/last_getopt_support.h +++ b/library/cpp/getopt/small/last_getopt_support.h @@ -1,36 +1,36 @@ #pragma once - + #include <util/string/cast.h> #include <util/generic/string.h> #include <util/generic/vector.h> -#include <util/generic/utility.h> +#include <util/generic/utility.h> #include <util/generic/yexception.h> - -namespace NLastGetopt { + +namespace NLastGetopt { class TOpt; class TOpts; class TOptsParser; class TOptsParseResult; - + /// base of all getopt exceptions class TException: public yexception { }; - + /// TOpts configuration is incorrect class TConfException: public TException { }; - + /// User passed incorrect arguments, parsing failed /// Note: use `throw TUsageException()` instead of `ythrow TUsageException()` to prevent appearence of stacktrace /// and location of the `ythrow` statment in error messages. class TUsageException: public TException { }; - + struct IOptHandler { virtual void HandleOpt(const TOptsParser* parser) = 0; virtual ~IOptHandler() = default; }; - + namespace NPrivate { template <typename TpFunc> class THandlerFunctor0 @@ -94,7 +94,7 @@ namespace NLastGetopt { class TStoreResultFunctor { private: T* Target_; - + public: TStoreResultFunctor(T* target) : Target_(target) @@ -128,7 +128,7 @@ namespace NLastGetopt { class TStoreValueFunctor { T* Target; const TpVal Value; - + public: template <typename TpArg> TStoreValueFunctor(T* target, const TpArg& value) @@ -155,7 +155,7 @@ namespace NLastGetopt { inline TStringBuf OptFromStringImpl<TStringBuf>(const TStringBuf& value) { return value; } - + template <> inline const char* OptFromStringImpl<const char*>(const TStringBuf& value) { return value.data(); diff --git a/library/cpp/getopt/small/opt.cpp b/library/cpp/getopt/small/opt.cpp index 2752987a55..744501765c 100644 --- a/library/cpp/getopt/small/opt.cpp +++ b/library/cpp/getopt/small/opt.cpp @@ -1,105 +1,105 @@ #include "opt.h" - + #include <util/system/progname.h> -#include <ctype.h> +#include <ctype.h> -using namespace NLastGetopt; +using namespace NLastGetopt; -namespace { +namespace { struct TOptsNoDefault: public TOpts { TOptsNoDefault(const TStringBuf& optstring = TStringBuf()) : TOpts(optstring) { } - }; - -} - -void Opt::Init(int argc, char* argv[], const char* optString, const Ion* longOptions, bool longOnly, bool isOpen) { - Ions_ = longOptions; - Err = true; - GotError_ = false; + }; + +} + +void Opt::Init(int argc, char* argv[], const char* optString, const Ion* longOptions, bool longOnly, bool isOpen) { + Ions_ = longOptions; + Err = true; + GotError_ = false; Ind = argc; - - Opts_.Reset(new TOptsNoDefault(optString)); + + Opts_.Reset(new TOptsNoDefault(optString)); for (const Ion* o = longOptions; o != nullptr && o->name != nullptr; ++o) { - TOpt* opt; + TOpt* opt; if ((unsigned)o->val < 0x80 && isalnum(o->val)) { - opt = &Opts_->CharOption(char(o->val)); + opt = &Opts_->CharOption(char(o->val)); opt->AddLongName(o->name); - } else { - Opts_->AddLongOption(o->name); - opt = const_cast<TOpt*>(&Opts_->GetLongOption(o->name)); - } - opt->HasArg_ = EHasArg(o->has_arg); + } else { + Opts_->AddLongOption(o->name); + opt = const_cast<TOpt*>(&Opts_->GetLongOption(o->name)); + } + opt->HasArg_ = EHasArg(o->has_arg); opt->UserValue(o); } - Opts_->AllowSingleDashForLong_ = longOnly; - Opts_->AllowPlusForLong_ = true; - Opts_->AllowUnknownCharOptions_ = isOpen; - Opts_->AllowUnknownLongOptions_ = false; + Opts_->AllowSingleDashForLong_ = longOnly; + Opts_->AllowPlusForLong_ = true; + Opts_->AllowUnknownCharOptions_ = isOpen; + Opts_->AllowUnknownLongOptions_ = false; + + OptsParser_.Reset(new TOptsParser(Opts_.Get(), argc, argv)); +} - OptsParser_.Reset(new TOptsParser(Opts_.Get(), argc, argv)); +Opt::Opt(int argc, char* argv[], const char* optString, const Ion* longOptions, bool longOnly, bool isOpen) { + Init(argc, argv, optString, longOptions, longOnly, isOpen); } -Opt::Opt(int argc, char* argv[], const char* optString, const Ion* longOptions, bool longOnly, bool isOpen) { - Init(argc, argv, optString, longOptions, longOnly, isOpen); -} - -Opt::Opt(int argc, const char* argv[], const char* optString, const Ion* longOptions, bool longOnly, bool isOpen) { +Opt::Opt(int argc, const char* argv[], const char* optString, const Ion* longOptions, bool longOnly, bool isOpen) { Init(argc, (char**)argv, optString, longOptions, longOnly, isOpen); -} - -int Opt::Get() { +} + +int Opt::Get() { return Get(nullptr); } -int Opt::Get(int* longOptionIndex) { - if (GotError_) - return EOF; +int Opt::Get(int* longOptionIndex) { + if (GotError_) + return EOF; Arg = nullptr; - try { - bool r = OptsParser_->Next(); + try { + bool r = OptsParser_->Next(); Ind = (int)OptsParser_->Pos_; - if (!r) { + if (!r) { return EOF; - } else { + } else { Arg = (char*)OptsParser_->CurVal(); if (!OptsParser_->CurOpt()) { - // possible if RETURN_IN_ORDER - return 1; - } else { + // possible if RETURN_IN_ORDER + return 1; + } else { const Ion* ion = (const Ion*)OptsParser_->CurOpt()->UserValue(); if (longOptionIndex) { *longOptionIndex = int(ion - Ions_); - } + } char c = OptsParser_->CurOpt()->GetCharOr0(); - return c != 0 ? c : ion->val; + return c != 0 ? c : ion->val; } } - } catch (const NLastGetopt::TException&) { - GotError_ = true; - if (Err) - Cerr << CurrentExceptionMessage() << Endl; - return '?'; + } catch (const NLastGetopt::TException&) { + GotError_ = true; + if (Err) + Cerr << CurrentExceptionMessage() << Endl; + return '?'; } } void Opt::DummyHelp(IOutputStream& os) { Opts_->PrintUsage(GetProgramName(), os); -} - -int Opt::GetArgC() const { +} + +int Opt::GetArgC() const { return (int)OptsParser_->Argc_; -} - -const char** Opt::GetArgV() const { - return OptsParser_->Argv_; -} - +} + +const char** Opt::GetArgV() const { + return OptsParser_->Argv_; +} + int opt_get_number(int& argc, char* argv[]) { int num = -1; for (int a = 1; a < argc; a++) { diff --git a/library/cpp/getopt/small/opt.h b/library/cpp/getopt/small/opt.h index 80701aa7e9..ecb57439bc 100644 --- a/library/cpp/getopt/small/opt.h +++ b/library/cpp/getopt/small/opt.h @@ -1,118 +1,118 @@ #pragma once - + #include "last_getopt.h" #include <util/generic/ptr.h> -#include <util/generic/noncopyable.h> - -// implementation of Opt class using last getopt - -/* - short-options syntax: - - opt-letter ::= - [^: ] - - opt-string ::= - '+'|'-'?({opt-letter}':'{0,2})* - - example: "AbCx:y:z::" - {A,b,C} options without argument - {x,y} options with argument - {z} option with optional argument - - 1. shortopts begins with '-' :=> RETURN_IN_ORDER - == non-option forces getopt to return 1 and to place non-option into optarg - - 2. shortopts begins with '+' :=> REQUIRE_ORDER +#include <util/generic/noncopyable.h> + +// implementation of Opt class using last getopt + +/* + short-options syntax: + + opt-letter ::= + [^: ] + + opt-string ::= + '+'|'-'?({opt-letter}':'{0,2})* + + example: "AbCx:y:z::" + {A,b,C} options without argument + {x,y} options with argument + {z} option with optional argument + + 1. shortopts begins with '-' :=> RETURN_IN_ORDER + == non-option forces getopt to return 1 and to place non-option into optarg + + 2. shortopts begins with '+' :=> REQUIRE_ORDER GetEnv(_POSIX_OPTION_ORDER) :=> REQUIRE_ORDER - == 1st non-option forces getopt to return EOF - - 3. default :=> PERMUTE - == exchange options with non-options and place all options first - - 4. '--' command line argument forces getopt to stop parsing and to return EOF - in any case - - long options should begin by '+' sign - or when (_getopt_long_only = 1) by '-' sign - - struct option { - char *name : option name - int has_arg: 0 | 1 | 2 = without | with | optional argument - int *flag : if (flag != 0) then getopt returns 0 and stores val into *flag - int val : if (flag == 0) then getopt returns val - } - - Example: - - struct option my_opts[] = { - { "delete", 0, &deletion_flag, DEL }, -- returns 0, deletion_flag := DEL - { "add", 1, NULL, 'a' }, -- returns 'a', argument in optarg - { NULL } - } -*/ - -#define OPT_RETURN_IN_ORDER "-" + == 1st non-option forces getopt to return EOF + + 3. default :=> PERMUTE + == exchange options with non-options and place all options first + + 4. '--' command line argument forces getopt to stop parsing and to return EOF + in any case + + long options should begin by '+' sign + or when (_getopt_long_only = 1) by '-' sign + + struct option { + char *name : option name + int has_arg: 0 | 1 | 2 = without | with | optional argument + int *flag : if (flag != 0) then getopt returns 0 and stores val into *flag + int val : if (flag == 0) then getopt returns val + } + + Example: + + struct option my_opts[] = { + { "delete", 0, &deletion_flag, DEL }, -- returns 0, deletion_flag := DEL + { "add", 1, NULL, 'a' }, -- returns 'a', argument in optarg + { NULL } + } +*/ + +#define OPT_RETURN_IN_ORDER "-" #define OPT_REQUIRE_ORDER "+" #define OPT_DONT_STORE_ARG ((void*)0) - + class Opt : TNonCopyable { -public: +public: enum HasArg { WithoutArg, WithArg, PossibleArg }; - - struct Ion { + + struct Ion { const char* name; HasArg has_arg; int* flag; int val; - }; - -private: - THolder<NLastGetopt::TOpts> Opts_; - THolder<NLastGetopt::TOptsParser> OptsParser_; - const Ion* Ions_; - bool GotError_; + }; + +private: + THolder<NLastGetopt::TOpts> Opts_; + THolder<NLastGetopt::TOptsParser> OptsParser_; + const Ion* Ions_; + bool GotError_; void Init(int argc, char* argv[], const char* optString, const Ion* longOptions = nullptr, bool longOnly = false, bool isOpen = false); - -public: + +public: Opt(int argc, char* argv[], const char* optString, const Ion* longOptions = nullptr, bool longOnly = false, bool isOpen = false); Opt(int argc, const char* argv[], const char* optString, const Ion* longOptions = nullptr, bool longOnly = false, bool isOpen = false); - - // Get() means next - int Get(); - int Get(int* longOptionIndex); + + // Get() means next + int Get(); + int Get(int* longOptionIndex); int operator()() { - return Get(); - } - + return Get(); + } + const char* GetArg() const { return Arg; } - + TVector<TString> GetFreeArgs() const { return NLastGetopt::TOptsParseResult(&*Opts_, GetArgC(), GetArgV()).GetFreeArgs(); } - // obsolete, use GetArg() instead + // obsolete, use GetArg() instead char* Arg; /* option argument if any or NULL */ - + int Ind; /* command line index */ bool Err; /* flag to print error messages */ - - int GetArgC() const; - const char** GetArgV() const; - + + int GetArgC() const; + const char** GetArgV() const; + void DummyHelp(IOutputStream& os = Cerr); -}; - -// call before getopt. returns non-negative int, removing it from arguments (not found: -1) -// Example: returns 11 for "progname -11abc", -1 for "progname -a11" +}; + +// call before getopt. returns non-negative int, removing it from arguments (not found: -1) +// Example: returns 11 for "progname -11abc", -1 for "progname -a11" int opt_get_number(int& argc, char* argv[]); - + #define OPTION_HANDLING_PROLOG \ { \ int optlet; \ @@ -128,12 +128,12 @@ int opt_get_number(int& argc, char* argv[]); #define OPTION_HANDLE_END \ } \ break; - + #define OPTION_HANDLE(opt, handle) \ OPTION_HANDLE_BEGIN(opt) \ handle; \ OPTION_HANDLE_END - + #define OPTION_HANDLING_EPILOG \ default: \ ythrow yexception() << "unknown optlet"; \ diff --git a/library/cpp/getopt/small/opt2.h b/library/cpp/getopt/small/opt2.h index c81cf345a4..4d9d943237 100644 --- a/library/cpp/getopt/small/opt2.h +++ b/library/cpp/getopt/small/opt2.h @@ -111,11 +111,11 @@ public: void AddError(const char* message = nullptr); public: - // non-option args + // non-option args TVector<char*> Pos; bool HasErrors; - -private: + +private: bool BadPosCount; char UnknownOption; char* UnknownLongOption; diff --git a/library/cpp/getopt/small/posix_getopt.cpp b/library/cpp/getopt/small/posix_getopt.cpp index cae224133a..bd06f3499f 100644 --- a/library/cpp/getopt/small/posix_getopt.cpp +++ b/library/cpp/getopt/small/posix_getopt.cpp @@ -2,76 +2,76 @@ #include <util/generic/ptr.h> -#include <ctype.h> - -namespace NLastGetopt { - char* optarg; - int optind; - int optopt; - int opterr; - int optreset; - - static THolder<TOpts> Opts; - static THolder<TOptsParser> OptsParser; - +#include <ctype.h> + +namespace NLastGetopt { + char* optarg; + int optind; + int optopt; + int opterr; + int optreset; + + static THolder<TOpts> Opts; + static THolder<TOptsParser> OptsParser; + int getopt_long_impl(int argc, char* const* argv, const char* optstring, const struct option* longopts, int* longindex, bool long_only) { - if (!Opts || optreset == 1) { + if (!Opts || optreset == 1) { optarg = nullptr; - optind = 1; - opterr = 1; - optreset = 0; - Opts.Reset(new TOpts(TOpts::Default(optstring))); - - Opts->AllowSingleDashForLong_ = long_only; - + optind = 1; + opterr = 1; + optreset = 0; + Opts.Reset(new TOpts(TOpts::Default(optstring))); + + Opts->AllowSingleDashForLong_ = long_only; + for (const struct option* o = longopts; o != nullptr && o->name != nullptr; ++o) { - TOpt* opt; + TOpt* opt; if ((unsigned)o->val < 0x80 && isalnum(o->val)) { - opt = &Opts->CharOption(char(o->val)); + opt = &Opts->CharOption(char(o->val)); opt->AddLongName(o->name); - } else { - Opts->AddLongOption(o->name); - opt = const_cast<TOpt*>(&Opts->GetLongOption(o->name)); - } - opt->HasArg_ = EHasArg(o->has_arg); + } else { + Opts->AddLongOption(o->name); + opt = const_cast<TOpt*>(&Opts->GetLongOption(o->name)); + } + opt->HasArg_ = EHasArg(o->has_arg); opt->UserValue(o->flag); - } - + } + OptsParser.Reset(new TOptsParser(&*Opts, argc, (const char**)argv)); - } - + } + optarg = nullptr; - - try { - if (!OptsParser->Next()) { - return -1; - } else { + + try { + if (!OptsParser->Next()) { + return -1; + } else { optarg = (char*)OptsParser->CurVal(); optind = (int)OptsParser->Pos_; if (longindex && OptsParser->CurOpt()) *longindex = (int)Opts->IndexOf(OptsParser->CurOpt()); return OptsParser->CurOpt() ? OptsParser->CurOpt()->GetCharOr0() : 1; - } - } catch (const NLastGetopt::TException&) { - return '?'; - } - } - + } + } catch (const NLastGetopt::TException&) { + return '?'; + } + } + int getopt_long(int argc, char* const* argv, const char* optstring, const struct option* longopts, int* longindex) { - return getopt_long_impl(argc, argv, optstring, longopts, longindex, false); - } - - int getopt_long_only(int argc, char* const* argv, const char* optstring, + return getopt_long_impl(argc, argv, optstring, longopts, longindex, false); + } + + int getopt_long_only(int argc, char* const* argv, const char* optstring, const struct option* longopts, int* longindex) { - return getopt_long_impl(argc, argv, optstring, longopts, longindex, true); - } - - // XXX: leading colon is not supported - // XXX: updating optind by client is not supported - int getopt(int argc, char* const* argv, const char* optstring) { + return getopt_long_impl(argc, argv, optstring, longopts, longindex, true); + } + + // XXX: leading colon is not supported + // XXX: updating optind by client is not supported + int getopt(int argc, char* const* argv, const char* optstring) { return getopt_long(argc, argv, optstring, nullptr, nullptr); - } - -} + } + +} diff --git a/library/cpp/getopt/small/posix_getopt.h b/library/cpp/getopt/small/posix_getopt.h index d26fa96641..e6af1e0284 100644 --- a/library/cpp/getopt/small/posix_getopt.h +++ b/library/cpp/getopt/small/posix_getopt.h @@ -1,32 +1,32 @@ #pragma once - -// implementation of posix getopt using last getopt for demonstration purposes - -#include "last_getopt.h" - -namespace NLastGetopt { - extern char* optarg; - extern int optind; - extern int optopt; - extern int opterr; - extern int optreset; - - enum { - no_argument = NO_ARGUMENT, - required_argument = REQUIRED_ARGUMENT, - optional_argument = OPTIONAL_ARGUMENT, - }; - - struct option { - const char* name; - int has_arg; - int* flag; - int val; - }; - - int getopt(int argc, char* const* argv, const char* optstring); + +// implementation of posix getopt using last getopt for demonstration purposes + +#include "last_getopt.h" + +namespace NLastGetopt { + extern char* optarg; + extern int optind; + extern int optopt; + extern int opterr; + extern int optreset; + + enum { + no_argument = NO_ARGUMENT, + required_argument = REQUIRED_ARGUMENT, + optional_argument = OPTIONAL_ARGUMENT, + }; + + struct option { + const char* name; + int has_arg; + int* flag; + int val; + }; + + int getopt(int argc, char* const* argv, const char* optstring); int getopt_long(int argc, char* const* argv, const char* optstring, const struct option* longopts, int* longindex); - int getopt_long_only(int argc, char* const* argv, const char* optstring, + int getopt_long_only(int argc, char* const* argv, const char* optstring, const struct option* longopts, int* longindex); -} +} diff --git a/library/cpp/getopt/ut/last_getopt_ut.cpp b/library/cpp/getopt/ut/last_getopt_ut.cpp index 7d6a29f2a5..c99a1d053d 100644 --- a/library/cpp/getopt/ut/last_getopt_ut.cpp +++ b/library/cpp/getopt/ut/last_getopt_ut.cpp @@ -2,22 +2,22 @@ #include <library/cpp/colorizer/colors.h> #include <library/cpp/testing/unittest/registar.h> - + #include <util/generic/array_size.h> #include <util/string/subst.h> #include <util/string/vector.h> #include <util/string/split.h> -using namespace NLastGetopt; - -namespace { +using namespace NLastGetopt; + +namespace { struct TOptsNoDefault: public TOpts { TOptsNoDefault(const TStringBuf& optstring = TStringBuf()) : TOpts(optstring) { } - }; - + }; + class TOptsParseResultTestWrapper: public TOptsParseResultException { TVector<const char*> Argv_; @@ -30,101 +30,101 @@ namespace { }; using V = TVector<const char*>; -} - -struct TOptsParserTester { - TOptsNoDefault Opts_; +} + +struct TOptsParserTester { + TOptsNoDefault Opts_; TVector<const char*> Argv_; - - THolder<TOptsParser> Parser_; - - void Initialize() { - if (!Parser_) + + THolder<TOptsParser> Parser_; + + void Initialize() { + if (!Parser_) Parser_.Reset(new TOptsParser(&Opts_, (int)Argv_.size(), Argv_.data())); - } - - void Accept() { - Initialize(); - UNIT_ASSERT(Parser_->Next()); - } - - void AcceptOption() { - Accept(); + } + + void Accept() { + Initialize(); + UNIT_ASSERT(Parser_->Next()); + } + + void AcceptOption() { + Accept(); UNIT_ASSERT(!!Parser_->CurOpt()); - } - - void AcceptOption(char c) { - AcceptOption(); + } + + void AcceptOption(char c) { + AcceptOption(); UNIT_ASSERT(Parser_->CurOpt()->CharIs(c)); - } - + } + void AcceptOption(const TString& optName) { - AcceptOption(); + AcceptOption(); UNIT_ASSERT(Parser_->CurOpt()->NameIs(optName)); - } - - template <typename TOpt> + } + + template <typename TOpt> void AcceptOptionWithValue(TOpt optName, const TString& value) { - AcceptOption(optName); + AcceptOption(optName); UNIT_ASSERT_VALUES_EQUAL_C(value, Parser_->CurValStr(), "; option " << optName); - } - - template <typename TOpt> - void AcceptOptionWithoutValue(TOpt optName) { - AcceptOption(optName); + } + + template <typename TOpt> + void AcceptOptionWithoutValue(TOpt optName) { + AcceptOption(optName); UNIT_ASSERT_C(!Parser_->CurVal(), ": opt " << optName << " must have no param"); - } - + } + void AcceptFreeArgInOrder(const TString& expected) { - Accept(); + Accept(); UNIT_ASSERT(!Parser_->CurOpt()); UNIT_ASSERT_VALUES_EQUAL(expected, Parser_->CurValStr()); - } - - size_t Pos_; - - void AcceptEndOfOptions() { - Initialize(); - UNIT_ASSERT(!Parser_->Next()); - Pos_ = Parser_->Pos_; - - // pos must not be changed after last meaningful invocation of Next() - UNIT_ASSERT(!Parser_->Next()); - UNIT_ASSERT_VALUES_EQUAL(Pos_, Parser_->Pos_); - UNIT_ASSERT(!Parser_->Next()); - UNIT_ASSERT_VALUES_EQUAL(Pos_, Parser_->Pos_); - } - - void AcceptError() { - Initialize(); - try { - Parser_->Next(); - UNIT_FAIL("expecting exception"); - } catch (const TUsageException&) { - // expecting - } - } - - void AcceptUnexpectedOption() { - Initialize(); - size_t pos = Parser_->Pos_; - size_t sop = Parser_->Sop_; - AcceptError(); - UNIT_ASSERT_VALUES_EQUAL(pos, Parser_->Pos_); - UNIT_ASSERT_VALUES_EQUAL(sop, Parser_->Sop_); - } - + } + + size_t Pos_; + + void AcceptEndOfOptions() { + Initialize(); + UNIT_ASSERT(!Parser_->Next()); + Pos_ = Parser_->Pos_; + + // pos must not be changed after last meaningful invocation of Next() + UNIT_ASSERT(!Parser_->Next()); + UNIT_ASSERT_VALUES_EQUAL(Pos_, Parser_->Pos_); + UNIT_ASSERT(!Parser_->Next()); + UNIT_ASSERT_VALUES_EQUAL(Pos_, Parser_->Pos_); + } + + void AcceptError() { + Initialize(); + try { + Parser_->Next(); + UNIT_FAIL("expecting exception"); + } catch (const TUsageException&) { + // expecting + } + } + + void AcceptUnexpectedOption() { + Initialize(); + size_t pos = Parser_->Pos_; + size_t sop = Parser_->Sop_; + AcceptError(); + UNIT_ASSERT_VALUES_EQUAL(pos, Parser_->Pos_); + UNIT_ASSERT_VALUES_EQUAL(sop, Parser_->Sop_); + } + void AcceptFreeArg(const TString& expected) { - UNIT_ASSERT(Pos_ < Parser_->Argc_); - UNIT_ASSERT_VALUES_EQUAL(expected, Parser_->Argv_[Pos_]); - ++Pos_; - } - - void AcceptEndOfFreeArgs() { - UNIT_ASSERT_VALUES_EQUAL(Argv_.size(), Pos_); - } -}; - + UNIT_ASSERT(Pos_ < Parser_->Argc_); + UNIT_ASSERT_VALUES_EQUAL(expected, Parser_->Argv_[Pos_]); + ++Pos_; + } + + void AcceptEndOfFreeArgs() { + UNIT_ASSERT_VALUES_EQUAL(Argv_.size(), Pos_); + } +}; + namespace { bool gSimpleFlag = false; void SimpleHander(void) { @@ -134,37 +134,37 @@ namespace { Y_UNIT_TEST_SUITE(TLastGetoptTests) { Y_UNIT_TEST(TestEqual) { - TOptsNoDefault opts; - opts.AddLongOption("from"); - opts.AddLongOption("to"); + TOptsNoDefault opts; + opts.AddLongOption("from"); + opts.AddLongOption("to"); TOptsParseResultTestWrapper r(&opts, V({"copy", "--from=/", "--to=/etc"})); - - UNIT_ASSERT_VALUES_EQUAL("copy", r.GetProgramName()); - UNIT_ASSERT_VALUES_EQUAL("/", r.Get("from")); - UNIT_ASSERT_VALUES_EQUAL("/etc", r.Get("to")); - UNIT_ASSERT_VALUES_EQUAL("/etc", r.GetOrElse("to", "trash")); - UNIT_ASSERT(r.Has("from")); - UNIT_ASSERT(r.Has("to")); + + UNIT_ASSERT_VALUES_EQUAL("copy", r.GetProgramName()); + UNIT_ASSERT_VALUES_EQUAL("/", r.Get("from")); + UNIT_ASSERT_VALUES_EQUAL("/etc", r.Get("to")); + UNIT_ASSERT_VALUES_EQUAL("/etc", r.GetOrElse("to", "trash")); + UNIT_ASSERT(r.Has("from")); + UNIT_ASSERT(r.Has("to")); UNIT_ASSERT_EXCEPTION(r.Get("left"), TException); - } - + } + Y_UNIT_TEST(TestCharOptions) { - TOptsNoDefault opts; - opts.AddCharOption('R', NO_ARGUMENT); - opts.AddCharOption('l', NO_ARGUMENT); - opts.AddCharOption('h', NO_ARGUMENT); + TOptsNoDefault opts; + opts.AddCharOption('R', NO_ARGUMENT); + opts.AddCharOption('l', NO_ARGUMENT); + opts.AddCharOption('h', NO_ARGUMENT); TOptsParseResultTestWrapper r(&opts, V({"cp", "/etc", "-Rl", "/tmp/etc"})); - UNIT_ASSERT(r.Has('R')); - UNIT_ASSERT(r.Has('l')); - UNIT_ASSERT(!r.Has('h')); - + UNIT_ASSERT(r.Has('R')); + UNIT_ASSERT(r.Has('l')); + UNIT_ASSERT(!r.Has('h')); + UNIT_ASSERT_VALUES_EQUAL(2u, r.GetFreeArgs().size()); UNIT_ASSERT_VALUES_EQUAL(2u, r.GetFreeArgCount()); - UNIT_ASSERT_VALUES_EQUAL("/etc", r.GetFreeArgs()[0]); - UNIT_ASSERT_VALUES_EQUAL("/tmp/etc", r.GetFreeArgs()[1]); - } - + UNIT_ASSERT_VALUES_EQUAL("/etc", r.GetFreeArgs()[0]); + UNIT_ASSERT_VALUES_EQUAL("/tmp/etc", r.GetFreeArgs()[1]); + } + Y_UNIT_TEST(TestFreeArgs) { TOptsNoDefault opts; opts.SetFreeArgsNum(1, 3); @@ -185,167 +185,167 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { } Y_UNIT_TEST(TestCharOptionsRequiredOptional) { - TOptsNoDefault opts; - opts.AddCharOption('d', REQUIRED_ARGUMENT); - opts.AddCharOption('e', REQUIRED_ARGUMENT); - opts.AddCharOption('x', REQUIRED_ARGUMENT); - opts.AddCharOption('y', REQUIRED_ARGUMENT); - opts.AddCharOption('l', NO_ARGUMENT); + TOptsNoDefault opts; + opts.AddCharOption('d', REQUIRED_ARGUMENT); + opts.AddCharOption('e', REQUIRED_ARGUMENT); + opts.AddCharOption('x', REQUIRED_ARGUMENT); + opts.AddCharOption('y', REQUIRED_ARGUMENT); + opts.AddCharOption('l', NO_ARGUMENT); TOptsParseResultTestWrapper r(&opts, V({"cmd", "-ld11", "-e", "22", "-lllx33", "-y", "44"})); - UNIT_ASSERT_VALUES_EQUAL("11", r.Get('d')); - UNIT_ASSERT_VALUES_EQUAL("22", r.Get('e')); - UNIT_ASSERT_VALUES_EQUAL("33", r.Get('x')); - UNIT_ASSERT_VALUES_EQUAL("44", r.Get('y')); - } - + UNIT_ASSERT_VALUES_EQUAL("11", r.Get('d')); + UNIT_ASSERT_VALUES_EQUAL("22", r.Get('e')); + UNIT_ASSERT_VALUES_EQUAL("33", r.Get('x')); + UNIT_ASSERT_VALUES_EQUAL("44", r.Get('y')); + } + Y_UNIT_TEST(TestReturnInOrder) { - TOptsParserTester tester; - tester.Opts_.AddLongOption('v', "value"); - tester.Opts_.ArgPermutation_ = RETURN_IN_ORDER; - - tester.Argv_.push_back("cmd"); - tester.Argv_.push_back("--value=11"); - tester.Argv_.push_back("xx"); - tester.Argv_.push_back("-v12"); - tester.Argv_.push_back("yy"); - tester.Argv_.push_back("--"); - tester.Argv_.push_back("-v13"); - tester.Argv_.push_back("--"); - - tester.AcceptOptionWithValue("value", "11"); - tester.AcceptFreeArgInOrder("xx"); - tester.AcceptOptionWithValue('v', "12"); - tester.AcceptFreeArgInOrder("yy"); - tester.AcceptFreeArgInOrder("-v13"); - tester.AcceptFreeArgInOrder("--"); - tester.AcceptEndOfOptions(); - tester.AcceptEndOfFreeArgs(); - } - + TOptsParserTester tester; + tester.Opts_.AddLongOption('v', "value"); + tester.Opts_.ArgPermutation_ = RETURN_IN_ORDER; + + tester.Argv_.push_back("cmd"); + tester.Argv_.push_back("--value=11"); + tester.Argv_.push_back("xx"); + tester.Argv_.push_back("-v12"); + tester.Argv_.push_back("yy"); + tester.Argv_.push_back("--"); + tester.Argv_.push_back("-v13"); + tester.Argv_.push_back("--"); + + tester.AcceptOptionWithValue("value", "11"); + tester.AcceptFreeArgInOrder("xx"); + tester.AcceptOptionWithValue('v', "12"); + tester.AcceptFreeArgInOrder("yy"); + tester.AcceptFreeArgInOrder("-v13"); + tester.AcceptFreeArgInOrder("--"); + tester.AcceptEndOfOptions(); + tester.AcceptEndOfFreeArgs(); + } + Y_UNIT_TEST(TestRequireOrder) { - TOptsParserTester tester; - tester.Opts_.ArgPermutation_ = REQUIRE_ORDER; - tester.Opts_.AddLongOption('v', "value"); - - tester.Argv_.push_back("cmd"); - tester.Argv_.push_back("--value=11"); - tester.Argv_.push_back("xx"); - tester.Argv_.push_back("-v12"); - tester.Argv_.push_back("yy"); - - tester.AcceptOptionWithValue("value", "11"); - tester.AcceptEndOfOptions(); - - tester.AcceptFreeArg("xx"); - tester.AcceptFreeArg("-v12"); - tester.AcceptFreeArg("yy"); - tester.AcceptEndOfFreeArgs(); - } - + TOptsParserTester tester; + tester.Opts_.ArgPermutation_ = REQUIRE_ORDER; + tester.Opts_.AddLongOption('v', "value"); + + tester.Argv_.push_back("cmd"); + tester.Argv_.push_back("--value=11"); + tester.Argv_.push_back("xx"); + tester.Argv_.push_back("-v12"); + tester.Argv_.push_back("yy"); + + tester.AcceptOptionWithValue("value", "11"); + tester.AcceptEndOfOptions(); + + tester.AcceptFreeArg("xx"); + tester.AcceptFreeArg("-v12"); + tester.AcceptFreeArg("yy"); + tester.AcceptEndOfFreeArgs(); + } + Y_UNIT_TEST(TestPlusForLongOption) { - TOptsParserTester tester; - tester.Opts_.AddLongOption('v', "value"); - tester.Opts_.AllowPlusForLong_ = true; - tester.Argv_.push_back("cmd"); - tester.Argv_.push_back("+value=11"); - tester.Argv_.push_back("xx"); - tester.Argv_.push_back("-v12"); - tester.Argv_.push_back("yy"); - - tester.AcceptOptionWithValue("value", "11"); - tester.AcceptOptionWithValue("value", "12"); - tester.AcceptEndOfOptions(); - - tester.AcceptFreeArg("xx"); - tester.AcceptFreeArg("yy"); - tester.AcceptEndOfFreeArgs(); - } - + TOptsParserTester tester; + tester.Opts_.AddLongOption('v', "value"); + tester.Opts_.AllowPlusForLong_ = true; + tester.Argv_.push_back("cmd"); + tester.Argv_.push_back("+value=11"); + tester.Argv_.push_back("xx"); + tester.Argv_.push_back("-v12"); + tester.Argv_.push_back("yy"); + + tester.AcceptOptionWithValue("value", "11"); + tester.AcceptOptionWithValue("value", "12"); + tester.AcceptEndOfOptions(); + + tester.AcceptFreeArg("xx"); + tester.AcceptFreeArg("yy"); + tester.AcceptEndOfFreeArgs(); + } + Y_UNIT_TEST(TestBug1) { - TOptsParserTester tester; - tester.Opts_.AddCharOptions("A:b:cd:"); - - tester.Argv_.push_back("cmd"); - tester.Argv_.push_back("-A"); - tester.Argv_.push_back("aaaa"); - tester.Argv_.push_back("zz"); - tester.Argv_.push_back("-c"); - tester.Argv_.push_back("-d8"); - tester.Argv_.push_back("ww"); - - tester.AcceptOptionWithValue('A', "aaaa"); - tester.AcceptOptionWithoutValue('c'); - tester.AcceptOptionWithValue('d', "8"); - tester.AcceptEndOfOptions(); - - tester.AcceptFreeArg("zz"); - tester.AcceptFreeArg("ww"); - tester.AcceptEndOfFreeArgs(); - } - + TOptsParserTester tester; + tester.Opts_.AddCharOptions("A:b:cd:"); + + tester.Argv_.push_back("cmd"); + tester.Argv_.push_back("-A"); + tester.Argv_.push_back("aaaa"); + tester.Argv_.push_back("zz"); + tester.Argv_.push_back("-c"); + tester.Argv_.push_back("-d8"); + tester.Argv_.push_back("ww"); + + tester.AcceptOptionWithValue('A', "aaaa"); + tester.AcceptOptionWithoutValue('c'); + tester.AcceptOptionWithValue('d', "8"); + tester.AcceptEndOfOptions(); + + tester.AcceptFreeArg("zz"); + tester.AcceptFreeArg("ww"); + tester.AcceptEndOfFreeArgs(); + } + Y_UNIT_TEST(TestPermuteComplex) { - TOptsParserTester tester; - - tester.Opts_.AddCharOption('x').NoArgument(); - tester.Opts_.AddCharOption('y').RequiredArgument(); - tester.Opts_.AddCharOption('z').NoArgument(); - tester.Opts_.AddCharOption('w').RequiredArgument(); - tester.Opts_.ArgPermutation_ = PERMUTE; - - tester.Argv_.push_back("cmd"); - tester.Argv_.push_back("-x"); - tester.Argv_.push_back("-y"); - tester.Argv_.push_back("val"); - tester.Argv_.push_back("freearg1"); - tester.Argv_.push_back("-zw"); - tester.Argv_.push_back("val2"); - tester.Argv_.push_back("freearg2"); - - tester.AcceptOptionWithoutValue('x'); - tester.AcceptOptionWithValue('y', "val"); - tester.AcceptOptionWithoutValue('z'); - tester.AcceptOptionWithValue('w', "val2"); - tester.AcceptEndOfOptions(); - tester.AcceptFreeArg("freearg1"); - tester.AcceptFreeArg("freearg2"); - tester.AcceptEndOfFreeArgs(); - } - + TOptsParserTester tester; + + tester.Opts_.AddCharOption('x').NoArgument(); + tester.Opts_.AddCharOption('y').RequiredArgument(); + tester.Opts_.AddCharOption('z').NoArgument(); + tester.Opts_.AddCharOption('w').RequiredArgument(); + tester.Opts_.ArgPermutation_ = PERMUTE; + + tester.Argv_.push_back("cmd"); + tester.Argv_.push_back("-x"); + tester.Argv_.push_back("-y"); + tester.Argv_.push_back("val"); + tester.Argv_.push_back("freearg1"); + tester.Argv_.push_back("-zw"); + tester.Argv_.push_back("val2"); + tester.Argv_.push_back("freearg2"); + + tester.AcceptOptionWithoutValue('x'); + tester.AcceptOptionWithValue('y', "val"); + tester.AcceptOptionWithoutValue('z'); + tester.AcceptOptionWithValue('w', "val2"); + tester.AcceptEndOfOptions(); + tester.AcceptFreeArg("freearg1"); + tester.AcceptFreeArg("freearg2"); + tester.AcceptEndOfFreeArgs(); + } + Y_UNIT_TEST(TestFinalDashDash) { - TOptsParserTester tester; - tester.Opts_.AddLongOption("size"); - - tester.Argv_.push_back("cmd"); - tester.Argv_.push_back("--"); - - tester.AcceptEndOfOptions(); - tester.AcceptEndOfFreeArgs(); - } - + TOptsParserTester tester; + tester.Opts_.AddLongOption("size"); + + tester.Argv_.push_back("cmd"); + tester.Argv_.push_back("--"); + + tester.AcceptEndOfOptions(); + tester.AcceptEndOfFreeArgs(); + } + Y_UNIT_TEST(TestDashDashAfterDashDash) { - TOptsParserTester tester; - tester.Opts_.AddLongOption("size"); - - tester.Argv_.push_back("cmd"); - tester.Argv_.push_back("--"); - tester.Argv_.push_back("--"); - tester.Argv_.push_back("--"); - - tester.AcceptEndOfOptions(); - tester.AcceptFreeArg("--"); - tester.AcceptFreeArg("--"); - tester.AcceptEndOfFreeArgs(); - } - + TOptsParserTester tester; + tester.Opts_.AddLongOption("size"); + + tester.Argv_.push_back("cmd"); + tester.Argv_.push_back("--"); + tester.Argv_.push_back("--"); + tester.Argv_.push_back("--"); + + tester.AcceptEndOfOptions(); + tester.AcceptFreeArg("--"); + tester.AcceptFreeArg("--"); + tester.AcceptEndOfFreeArgs(); + } + Y_UNIT_TEST(TestUnexpectedUnknownOption) { - TOptsParserTester tester; - - tester.Argv_.push_back("cmd"); - tester.Argv_.push_back("-x"); - - tester.AcceptUnexpectedOption(); - } - + TOptsParserTester tester; + + tester.Argv_.push_back("cmd"); + tester.Argv_.push_back("-x"); + + tester.AcceptUnexpectedOption(); + } + Y_UNIT_TEST(TestDuplicatedOptionCrash) { // this test is broken, cause UNIT_ASSERT(false) always throws return; @@ -364,84 +364,84 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { } Y_UNIT_TEST(TestPositionWhenNoArgs) { - TOptsParserTester tester; - - tester.Argv_.push_back("cmd"); - - tester.Opts_.AddCharOption('c'); - - tester.AcceptEndOfOptions(); - + TOptsParserTester tester; + + tester.Argv_.push_back("cmd"); + + tester.Opts_.AddCharOption('c'); + + tester.AcceptEndOfOptions(); + UNIT_ASSERT_VALUES_EQUAL(1u, tester.Parser_->Pos_); - } - + } + Y_UNIT_TEST(TestExpectedUnknownCharOption) { - TOptsParserTester tester; - - tester.Argv_.push_back("cmd"); - tester.Argv_.push_back("-x"); - tester.Argv_.push_back("-y"); - tester.Argv_.push_back("val"); - tester.Argv_.push_back("freearg1"); - tester.Argv_.push_back("-zw"); - tester.Argv_.push_back("val2"); - tester.Argv_.push_back("freearg2"); - - tester.Opts_.AllowUnknownCharOptions_ = true; - - tester.AcceptOptionWithoutValue('x'); - tester.AcceptOptionWithValue('y', "val"); - tester.AcceptOptionWithoutValue('z'); - tester.AcceptOptionWithValue('w', "val2"); - tester.AcceptEndOfOptions(); - tester.AcceptFreeArg("freearg1"); - tester.AcceptFreeArg("freearg2"); - tester.AcceptEndOfFreeArgs(); - } - -#if 0 + TOptsParserTester tester; + + tester.Argv_.push_back("cmd"); + tester.Argv_.push_back("-x"); + tester.Argv_.push_back("-y"); + tester.Argv_.push_back("val"); + tester.Argv_.push_back("freearg1"); + tester.Argv_.push_back("-zw"); + tester.Argv_.push_back("val2"); + tester.Argv_.push_back("freearg2"); + + tester.Opts_.AllowUnknownCharOptions_ = true; + + tester.AcceptOptionWithoutValue('x'); + tester.AcceptOptionWithValue('y', "val"); + tester.AcceptOptionWithoutValue('z'); + tester.AcceptOptionWithValue('w', "val2"); + tester.AcceptEndOfOptions(); + tester.AcceptFreeArg("freearg1"); + tester.AcceptFreeArg("freearg2"); + tester.AcceptEndOfFreeArgs(); + } + +#if 0 Y_UNIT_TEST(TestRequiredParams) { - TOptsParserTester tester; - - tester.Argv_.push_back("cmd"); - tester.Argv_.push_back("--port=1231"); - tester.Argv_.push_back("asas"); - - tester.Opts_.AddLongOption("port"); - tester.Opts_.AddLongOption("home").Required(); - - tester.AcceptOptionWithValue("port", "1231"); - tester.AcceptError(); - } -#endif - + TOptsParserTester tester; + + tester.Argv_.push_back("cmd"); + tester.Argv_.push_back("--port=1231"); + tester.Argv_.push_back("asas"); + + tester.Opts_.AddLongOption("port"); + tester.Opts_.AddLongOption("home").Required(); + + tester.AcceptOptionWithValue("port", "1231"); + tester.AcceptError(); + } +#endif + Y_UNIT_TEST(TestStoreResult) { - TOptsNoDefault opts; + TOptsNoDefault opts; TString data; - int number; + int number; TMaybe<TString> optionalString0, optionalString1; TMaybe<int> optionalNumber0, optionalNumber1; - opts.AddLongOption('d', "data").StoreResult(&data); - opts.AddLongOption('n', "number").StoreResult(&number); + opts.AddLongOption('d', "data").StoreResult(&data); + opts.AddLongOption('n', "number").StoreResult(&number); opts.AddLongOption("optional-string-0").StoreResult(&optionalString0); opts.AddLongOption("optional-number-0").StoreResult(&optionalNumber0); opts.AddLongOption("optional-string-1").StoreResult(&optionalString1); opts.AddLongOption("optional-number-1").StoreResult(&optionalNumber1); TOptsParseResultTestWrapper r(&opts, V({"cmd", "--data=jjhh", "-n", "11", "--optional-number-1=8", "--optional-string-1=os1"})); - UNIT_ASSERT_VALUES_EQUAL("jjhh", data); - UNIT_ASSERT_VALUES_EQUAL(11, number); + UNIT_ASSERT_VALUES_EQUAL("jjhh", data); + UNIT_ASSERT_VALUES_EQUAL(11, number); UNIT_ASSERT(!optionalString0.Defined()); UNIT_ASSERT(!optionalNumber0.Defined()); UNIT_ASSERT_VALUES_EQUAL(*optionalString1, "os1"); UNIT_ASSERT_VALUES_EQUAL(*optionalNumber1, 8); - } - + } + Y_UNIT_TEST(TestStoreValue) { int a = 0, b = 0; size_t c = 0; EHasArg e = NO_ARGUMENT; - TOptsNoDefault opts; + TOptsNoDefault opts; opts.AddLongOption('a', "alpha").NoArgument().StoreValue(&a, 42); opts.AddLongOption('b', "beta").NoArgument().StoreValue(&b, 24); opts.AddLongOption('e', "enum").NoArgument().StoreValue(&e, REQUIRED_ARGUMENT).StoreValue(&c, 12345); @@ -472,17 +472,17 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { } Y_UNIT_TEST(TestDefaultValue) { - TOptsNoDefault opts; - opts.AddLongOption("path").DefaultValue("/etc"); + TOptsNoDefault opts; + opts.AddLongOption("path").DefaultValue("/etc"); int value = 42; opts.AddLongOption("value").StoreResult(&value).DefaultValue(32); TOptsParseResultTestWrapper r(&opts, V({"cmd", "dfdf"})); - UNIT_ASSERT_VALUES_EQUAL("/etc", r.Get("path")); + UNIT_ASSERT_VALUES_EQUAL("/etc", r.Get("path")); UNIT_ASSERT_VALUES_EQUAL(32, value); - } - + } + Y_UNIT_TEST(TestSplitValue) { - TOptsNoDefault opts; + TOptsNoDefault opts; TVector<TString> vals; opts.AddLongOption('s', "split").SplitHandler(&vals, ','); TOptsParseResultTestWrapper r(&opts, V({"prog", "--split=a,b,c"})); @@ -508,17 +508,17 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { } Y_UNIT_TEST(TestParseArgs) { - TOptsNoDefault o("AbCx:y:z::"); - UNIT_ASSERT_EQUAL(o.GetCharOption('A').HasArg_, NO_ARGUMENT); - UNIT_ASSERT_EQUAL(o.GetCharOption('b').HasArg_, NO_ARGUMENT); - UNIT_ASSERT_EQUAL(o.GetCharOption('C').HasArg_, NO_ARGUMENT); - UNIT_ASSERT_EQUAL(o.GetCharOption('x').HasArg_, REQUIRED_ARGUMENT); - UNIT_ASSERT_EQUAL(o.GetCharOption('y').HasArg_, REQUIRED_ARGUMENT); - UNIT_ASSERT_EQUAL(o.GetCharOption('z').HasArg_, OPTIONAL_ARGUMENT); - } - + TOptsNoDefault o("AbCx:y:z::"); + UNIT_ASSERT_EQUAL(o.GetCharOption('A').HasArg_, NO_ARGUMENT); + UNIT_ASSERT_EQUAL(o.GetCharOption('b').HasArg_, NO_ARGUMENT); + UNIT_ASSERT_EQUAL(o.GetCharOption('C').HasArg_, NO_ARGUMENT); + UNIT_ASSERT_EQUAL(o.GetCharOption('x').HasArg_, REQUIRED_ARGUMENT); + UNIT_ASSERT_EQUAL(o.GetCharOption('y').HasArg_, REQUIRED_ARGUMENT); + UNIT_ASSERT_EQUAL(o.GetCharOption('z').HasArg_, OPTIONAL_ARGUMENT); + } + Y_UNIT_TEST(TestRequiredOpts) { - TOptsNoDefault opts; + TOptsNoDefault opts; TOpt& opt_d = opts.AddCharOption('d'); // test 'not required' @@ -549,14 +549,14 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { }; Y_UNIT_TEST(TestHandlers) { { - TOptsNoDefault opts; + TOptsNoDefault opts; bool flag = false; opts.AddLongOption("flag").Handler0(HandlerStoreTrue(&flag)).NoArgument(); TOptsParseResultTestWrapper r(&opts, V({"cmd", "--flag"})); UNIT_ASSERT(flag); } { - TOptsNoDefault opts; + TOptsNoDefault opts; unsigned uval = 5; double fval = 0.0; opts.AddLongOption("flag1").RequiredArgument().StoreResult(&uval); @@ -702,17 +702,17 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { Y_UNIT_TEST(TestAppendTo) { TVector<int> ints; - - TOptsNoDefault opts; - opts.AddLongOption("size").AppendTo(&ints); - + + TOptsNoDefault opts; + opts.AddLongOption("size").AppendTo(&ints); + TOptsParseResultTestWrapper r(&opts, V({"cmd", "--size=17", "--size=19"})); - - UNIT_ASSERT_VALUES_EQUAL(size_t(2), ints.size()); - UNIT_ASSERT_VALUES_EQUAL(17, ints.at(0)); - UNIT_ASSERT_VALUES_EQUAL(19, ints.at(1)); - } - + + UNIT_ASSERT_VALUES_EQUAL(size_t(2), ints.size()); + UNIT_ASSERT_VALUES_EQUAL(17, ints.at(0)); + UNIT_ASSERT_VALUES_EQUAL(19, ints.at(1)); + } + Y_UNIT_TEST(TestEmplaceTo) { TVector<std::tuple<TString>> richPaths; @@ -791,4 +791,4 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { UNIT_ASSERT_VALUES_EQUAL(25, number); UNIT_ASSERT_VALUES_EQUAL(2, r.GetFreeArgCount()); } -} +} diff --git a/library/cpp/getopt/ut/opt2_ut.cpp b/library/cpp/getopt/ut/opt2_ut.cpp index facf0a6b0d..0e7464747c 100644 --- a/library/cpp/getopt/ut/opt2_ut.cpp +++ b/library/cpp/getopt/ut/opt2_ut.cpp @@ -1,13 +1,13 @@ #include <library/cpp/getopt/opt2.h> #include <library/cpp/testing/unittest/registar.h> - + //using namespace NLastGetopt; - + Y_UNIT_TEST_SUITE(Opt2Test) { Y_UNIT_TEST(TestSimple) { int argc = 8; - char* argv[] = { + char* argv[] = { (char*)"cmd", (char*)"--aaaa=aaaa", (char*)"zz", @@ -16,10 +16,10 @@ Y_UNIT_TEST_SUITE(Opt2Test) { (char*)"-c", (char*)"-d8", (char*)"ww", - }; - + }; + Opt2 opt(argc, argv, "A:b:cd:e:x:", 2, "aaaa=A"); - + const char* edef = "edef"; const char* a = opt.Arg('A', "<var_name> - usage of -A"); int b = opt.Int('b', "<var_name> - usage of -b", 2); @@ -27,21 +27,21 @@ Y_UNIT_TEST_SUITE(Opt2Test) { int d = opt.Int('d', "<var_name> - usage of -d", 13); const char* e = opt.Arg('e', "<unused> - only default is really used", edef); const TVector<const char*>& x = opt.MArg('x', "<var_name> - usage of -x"); - - UNIT_ASSERT(!opt.AutoUsage("<L> <M>")); - UNIT_ASSERT_VALUES_EQUAL("aaaa", a); - UNIT_ASSERT_VALUES_EQUAL(2, b); - UNIT_ASSERT(c); - UNIT_ASSERT_VALUES_EQUAL(8, d); + + UNIT_ASSERT(!opt.AutoUsage("<L> <M>")); + UNIT_ASSERT_VALUES_EQUAL("aaaa", a); + UNIT_ASSERT_VALUES_EQUAL(2, b); + UNIT_ASSERT(c); + UNIT_ASSERT_VALUES_EQUAL(8, d); UNIT_ASSERT_VALUES_EQUAL((void*)edef, e); - + UNIT_ASSERT_VALUES_EQUAL(2u, opt.Pos.size()); UNIT_ASSERT_STRINGS_EQUAL("zz", opt.Pos.at(0)); UNIT_ASSERT_VALUES_EQUAL((void*)argv[2], opt.Pos.at(0)); UNIT_ASSERT_STRINGS_EQUAL("ww", opt.Pos.at(1)); UNIT_ASSERT_STRINGS_EQUAL("1", x.at(0)); UNIT_ASSERT_STRINGS_EQUAL("2", x.at(1)); - } + } Y_UNIT_TEST(TestErrors1) { int argc = 4; @@ -60,4 +60,4 @@ Y_UNIT_TEST_SUITE(Opt2Test) { UNIT_ASSERT(c); UNIT_ASSERT_VALUES_EQUAL((void*)edef, e); } -} +} diff --git a/library/cpp/getopt/ut/opt_ut.cpp b/library/cpp/getopt/ut/opt_ut.cpp index 66122f6ae3..441aa493a0 100644 --- a/library/cpp/getopt/ut/opt_ut.cpp +++ b/library/cpp/getopt/ut/opt_ut.cpp @@ -2,19 +2,19 @@ #include <library/cpp/testing/unittest/registar.h> #include <util/string/vector.h> - + Y_UNIT_TEST_SUITE(OptTest) { Y_UNIT_TEST(TestSimple) { - int argc = 3; - char* argv[] = { + int argc = 3; + char* argv[] = { (char*)"cmd", (char*)"-x"}; - Opt opt(argc, argv, ""); - opt.Err = false; // be quiet - UNIT_ASSERT_VALUES_EQUAL('?', opt.Get()); - UNIT_ASSERT_VALUES_EQUAL(EOF, opt.Get()); - UNIT_ASSERT_VALUES_EQUAL(EOF, opt.Get()); - UNIT_ASSERT_VALUES_EQUAL(EOF, opt.Get()); - } + Opt opt(argc, argv, ""); + opt.Err = false; // be quiet + UNIT_ASSERT_VALUES_EQUAL('?', opt.Get()); + UNIT_ASSERT_VALUES_EQUAL(EOF, opt.Get()); + UNIT_ASSERT_VALUES_EQUAL(EOF, opt.Get()); + UNIT_ASSERT_VALUES_EQUAL(EOF, opt.Get()); + } Y_UNIT_TEST(TestFreeArguments) { Opt::Ion options[] = { @@ -49,4 +49,4 @@ Y_UNIT_TEST_SUITE(OptTest) { UNIT_ASSERT_VALUES_EQUAL(optionValue, "ARG2"); } } -} +} diff --git a/library/cpp/getopt/ut/posix_getopt_ut.cpp b/library/cpp/getopt/ut/posix_getopt_ut.cpp index 9235e7340d..b6d374bf28 100644 --- a/library/cpp/getopt/ut/posix_getopt_ut.cpp +++ b/library/cpp/getopt/ut/posix_getopt_ut.cpp @@ -1,119 +1,119 @@ #include <library/cpp/getopt/posix_getopt.h> #include <library/cpp/testing/unittest/registar.h> - -using namespace NLastGetopt; - + +using namespace NLastGetopt; + Y_UNIT_TEST_SUITE(TPosixGetoptTest) { Y_UNIT_TEST(TestSimple) { - int argc = 6; + int argc = 6; const char* argv0[] = {"program", "-b", "-f1", "-f", "2", "zzzz"}; char** const argv = (char**)argv0; - - NLastGetopt::optreset = 1; - UNIT_ASSERT_VALUES_EQUAL('b', NLastGetopt::getopt(argc, argv, "bf:")); - UNIT_ASSERT_VALUES_EQUAL('f', NLastGetopt::getopt(argc, argv, "bf:")); + + NLastGetopt::optreset = 1; + UNIT_ASSERT_VALUES_EQUAL('b', NLastGetopt::getopt(argc, argv, "bf:")); + UNIT_ASSERT_VALUES_EQUAL('f', NLastGetopt::getopt(argc, argv, "bf:")); UNIT_ASSERT_VALUES_EQUAL(NLastGetopt::optarg, TString("1")); - UNIT_ASSERT_VALUES_EQUAL('f', NLastGetopt::getopt(argc, argv, "bf:")); + UNIT_ASSERT_VALUES_EQUAL('f', NLastGetopt::getopt(argc, argv, "bf:")); UNIT_ASSERT_VALUES_EQUAL(NLastGetopt::optarg, TString("2")); - UNIT_ASSERT_VALUES_EQUAL(-1, NLastGetopt::getopt(argc, argv, "bf:")); - - UNIT_ASSERT_VALUES_EQUAL(5, NLastGetopt::optind); - } - + UNIT_ASSERT_VALUES_EQUAL(-1, NLastGetopt::getopt(argc, argv, "bf:")); + + UNIT_ASSERT_VALUES_EQUAL(5, NLastGetopt::optind); + } + Y_UNIT_TEST(TestLong) { - int daggerset = 0; - /* options descriptor */ - const NLastGetopt::option longopts[] = { + int daggerset = 0; + /* options descriptor */ + const NLastGetopt::option longopts[] = { {"buffy", no_argument, nullptr, 'b'}, {"fluoride", required_argument, nullptr, 'f'}, {"daggerset", no_argument, &daggerset, 1}, {nullptr, 0, nullptr, 0}}; - - int argc = 7; + + int argc = 7; const char* argv0[] = {"program", "-b", "--buffy", "-f1", "--fluoride=2", "--daggerset", "zzzz"}; char** const argv = (char**)argv0; - - int longIndex; - - NLastGetopt::optreset = 1; - UNIT_ASSERT_VALUES_EQUAL('b', NLastGetopt::getopt_long(argc, argv, "bf:", longopts, &longIndex)); - UNIT_ASSERT_VALUES_EQUAL(0, longIndex); + + int longIndex; + + NLastGetopt::optreset = 1; + UNIT_ASSERT_VALUES_EQUAL('b', NLastGetopt::getopt_long(argc, argv, "bf:", longopts, &longIndex)); + UNIT_ASSERT_VALUES_EQUAL(0, longIndex); UNIT_ASSERT_VALUES_EQUAL('b', NLastGetopt::getopt_long(argc, argv, "bf:", longopts, nullptr)); - UNIT_ASSERT_VALUES_EQUAL('f', NLastGetopt::getopt_long(argc, argv, "bf:", longopts, &longIndex)); - UNIT_ASSERT_VALUES_EQUAL(1, longIndex); + UNIT_ASSERT_VALUES_EQUAL('f', NLastGetopt::getopt_long(argc, argv, "bf:", longopts, &longIndex)); + UNIT_ASSERT_VALUES_EQUAL(1, longIndex); UNIT_ASSERT_VALUES_EQUAL('f', NLastGetopt::getopt_long(argc, argv, "bf:", longopts, nullptr)); UNIT_ASSERT_VALUES_EQUAL(0, NLastGetopt::getopt_long(argc, argv, "bf:", longopts, nullptr)); UNIT_ASSERT_VALUES_EQUAL(-1, NLastGetopt::getopt_long(argc, argv, "bf:", longopts, nullptr)); - - UNIT_ASSERT_VALUES_EQUAL(6, NLastGetopt::optind); - } - + + UNIT_ASSERT_VALUES_EQUAL(6, NLastGetopt::optind); + } + Y_UNIT_TEST(TestLongPermutation) { - int daggerset = 0; - /* options descriptor */ - const NLastGetopt::option longopts[] = { + int daggerset = 0; + /* options descriptor */ + const NLastGetopt::option longopts[] = { {"buffy", no_argument, nullptr, 'b'}, {"fluoride", required_argument, nullptr, 'f'}, {"daggerset", no_argument, &daggerset, 1}, {nullptr, 0, nullptr, 0}}; - - int argc = 7; + + int argc = 7; const char* argv0[] = {"program", "aa", "-b", "bb", "cc", "--buffy", "dd"}; char** const argv = (char**)argv0; - - NLastGetopt::optreset = 1; + + NLastGetopt::optreset = 1; UNIT_ASSERT_VALUES_EQUAL('b', NLastGetopt::getopt_long(argc, argv, "bf:", longopts, nullptr)); UNIT_ASSERT_VALUES_EQUAL('b', NLastGetopt::getopt_long(argc, argv, "bf:", longopts, nullptr)); UNIT_ASSERT_VALUES_EQUAL(-1, NLastGetopt::getopt_long(argc, argv, "bf:", longopts, nullptr)); - - UNIT_ASSERT_VALUES_EQUAL(3, NLastGetopt::optind); - } - + + UNIT_ASSERT_VALUES_EQUAL(3, NLastGetopt::optind); + } + Y_UNIT_TEST(TestNoOptionsOptionsWithDoubleDash) { - const NLastGetopt::option longopts[] = { + const NLastGetopt::option longopts[] = { {"buffy", no_argument, nullptr, 'b'}, {"fluoride", no_argument, nullptr, 'f'}, {nullptr, 0, nullptr, 0}}; - - int argc = 2; + + int argc = 2; const char* argv0[] = {"program", "--bf"}; char** const argv = (char**)argv0; - - NLastGetopt::optreset = 1; + + NLastGetopt::optreset = 1; UNIT_ASSERT_VALUES_EQUAL('?', NLastGetopt::getopt_long(argc, argv, "bf", longopts, nullptr)); - } - + } + Y_UNIT_TEST(TestLongOnly) { - const NLastGetopt::option longopts[] = { + const NLastGetopt::option longopts[] = { {"foo", no_argument, nullptr, 'F'}, {"fluoride", no_argument, nullptr, 'f'}, {"ogogo", no_argument, nullptr, 'o'}, {nullptr, 0, nullptr, 0}}; - - int argc = 4; + + int argc = 4; const char* argv0[] = {"program", "--foo", "-foo", "-fo"}; char** const argv = (char**)argv0; - - NLastGetopt::optreset = 1; + + NLastGetopt::optreset = 1; UNIT_ASSERT_VALUES_EQUAL('F', NLastGetopt::getopt_long_only(argc, argv, "fo", longopts, nullptr)); UNIT_ASSERT_VALUES_EQUAL('F', NLastGetopt::getopt_long_only(argc, argv, "fo", longopts, nullptr)); UNIT_ASSERT_VALUES_EQUAL('f', NLastGetopt::getopt_long_only(argc, argv, "fo", longopts, nullptr)); UNIT_ASSERT_VALUES_EQUAL('o', NLastGetopt::getopt_long_only(argc, argv, "fo", longopts, nullptr)); UNIT_ASSERT_VALUES_EQUAL(-1, NLastGetopt::getopt_long_only(argc, argv, "fo", longopts, nullptr)); - } - + } + Y_UNIT_TEST(TestLongWithoutOnlySingleDashNowAllowed) { - const NLastGetopt::option longopts[] = { + const NLastGetopt::option longopts[] = { {"foo", no_argument, nullptr, 'F'}, {"zoo", no_argument, nullptr, 'z'}, {nullptr, 0, nullptr, 0}}; - - int argc = 2; + + int argc = 2; const char* argv0[] = {"program", "-foo"}; char** const argv = (char**)argv0; - - NLastGetopt::optreset = 1; + + NLastGetopt::optreset = 1; UNIT_ASSERT_VALUES_EQUAL('?', NLastGetopt::getopt_long(argc, argv, "z", longopts, nullptr)); - } -} + } +} diff --git a/library/cpp/http/fetch/httpfsm.h b/library/cpp/http/fetch/httpfsm.h index e79fd7a152..c4abdcd0d2 100644 --- a/library/cpp/http/fetch/httpfsm.h +++ b/library/cpp/http/fetch/httpfsm.h @@ -3,7 +3,7 @@ #include "httpheader.h" #include <util/system/maxlen.h> -#include <util/datetime/parser.h> +#include <util/datetime/parser.h> #include <time.h> @@ -19,10 +19,10 @@ struct THttpHeaderParser { return execute((unsigned char*)inBuf, (int)len); } - int Execute(TStringBuf str) { - return Execute(str.data(), str.size()); - } - + int Execute(TStringBuf str) { + return Execute(str.data(), str.size()); + } + int Init(THttpHeader* h) { int ret = Init((THttpBaseHeader*)(h)); hd = h; diff --git a/library/cpp/http/fetch/httpfsm.rl6 b/library/cpp/http/fetch/httpfsm.rl6 index 2712c013bb..eab0328b18 100644 --- a/library/cpp/http/fetch/httpfsm.rl6 +++ b/library/cpp/http/fetch/httpfsm.rl6 @@ -4,7 +4,7 @@ #include <library/cpp/charset/doccodes.h> #include <library/cpp/charset/codepage.h> #include <library/cpp/http/misc/httpcodes.h> -#include <util/datetime/base.h> +#include <util/datetime/base.h> #include <util/generic/ylimits.h> #include <algorithm> // max @@ -15,8 +15,8 @@ #pragma warning(disable: 4702) // unreachable code #endif -#define c(i) I = i; -#define m(i) I = std::max(I, (long)i); +#define c(i) I = i; +#define m(i) I = std::max(I, (long)i); static inline int X(unsigned char c) { return (c >= 'A' ? ((c & 0xdf) - 'A' + 10) : (c - '0')); @@ -40,7 +40,7 @@ static inline void setguarded(x &val, long cnt) { machine http_header_parser; include HttpDateTimeParser "../../../../util/datetime/parser.rl6"; - + alphtype unsigned char; ################# 2.2 Basic Rules ################# @@ -61,8 +61,8 @@ text = (text_char | lw)*; any_text = (any_text_char | lw)*; def = lws ':' lws; -action clear_buf { buflen = 0; } -action update_buf { if (buflen < sizeof(buf)) buf[buflen++] = fc; } +action clear_buf { buflen = 0; } +action update_buf { if (buflen < sizeof(buf)) buf[buflen++] = fc; } ################################################### ############ response status line ################# @@ -123,14 +123,14 @@ action set_connection { setguarded(base_hd->connection_closed, I); } c_token = "close"i %{m(1)} | "keep-alive"i %{m(0)}; c_tokenlist = c_token (lws ',' lws c_token)?; -connection = "connection"i def %beg_connection c_tokenlist eoh %set_connection; +connection = "connection"i def %beg_connection c_tokenlist eoh %set_connection; ################# content-encoding ################ -action beg_content_encoding { I = HTTP_COMPRESSION_ERROR; } +action beg_content_encoding { I = HTTP_COMPRESSION_ERROR; } action set_content_encoding { base_hd->compression_method = ((base_hd->compression_method == HTTP_COMPRESSION_UNSET || base_hd->compression_method == I) ? - I : (int)HTTP_COMPRESSION_ERROR); } + I : (int)HTTP_COMPRESSION_ERROR); } ce_tokenlist = "identity"i %{c(HTTP_COMPRESSION_IDENTITY)} | "gzip"i %{c(HTTP_COMPRESSION_GZIP)} @@ -138,7 +138,7 @@ ce_tokenlist = "identity"i %{c(HTTP_COMPRESSION_IDENTITY)} | "deflate"i %{c(HTTP_COMPRESSION_DEFLATE)} | "compress"i %{c(HTTP_COMPRESSION_COMPRESS)} | "x-compress"i %{c(HTTP_COMPRESSION_COMPRESS)}; -content_encoding = "content-encoding"i def %beg_content_encoding ce_tokenlist eoh %set_content_encoding; +content_encoding = "content-encoding"i def %beg_content_encoding ce_tokenlist eoh %set_content_encoding; ################# transfer-encoding ############### action beg_encoding { guard(base_hd->transfer_chunked); } @@ -146,13 +146,13 @@ action set_encoding { setguarded(base_hd->transfer_chunked, I); } e_tokenlist = "identity"i %{c(0)} | "chunked"i %{c(1)}; -transfer_encoding = "transfer-encoding"i def %beg_encoding e_tokenlist eoh %set_encoding; +transfer_encoding = "transfer-encoding"i def %beg_encoding e_tokenlist eoh %set_encoding; ################# content-length ################## action beg_content_length { guard(base_hd->content_length); } action set_content_length { setguarded(base_hd->content_length, I); } -content_length = "content-length"i def %beg_content_length int eoh %set_content_length; +content_length = "content-length"i def %beg_content_length int eoh %set_content_length; ################# content-range ################### action beg_content_range_start { guard(base_hd->content_range_start); I = -1; } @@ -162,9 +162,9 @@ action set_content_range_end { setguarded(base_hd->content_range_end, I); } action beg_content_range_el { guard(base_hd->content_range_entity_length); I = -1; } action set_content_range_el { setguarded(base_hd->content_range_entity_length, I); } -content_range = "content-range"i def "bytes"i sp %beg_content_range_start int '-' %set_content_range_start - %beg_content_range_end int '/' %set_content_range_end - %beg_content_range_el int eoh %set_content_range_el; +content_range = "content-range"i def "bytes"i sp %beg_content_range_start int '-' %set_content_range_start + %beg_content_range_end int '/' %set_content_range_end + %beg_content_range_el int eoh %set_content_range_el; ################# accept-ranges ################### action beg_accept_ranges { @@ -177,12 +177,12 @@ action set_accept_ranges { if (hd) setguarded(hd->accept_ranges, I); } ar_tokenlist = "bytes"i %{c(1)} | "none"i %{c(0)}; -accept_ranges = "accept-ranges"i def %beg_accept_ranges ar_tokenlist eoh %set_accept_ranges; +accept_ranges = "accept-ranges"i def %beg_accept_ranges ar_tokenlist eoh %set_accept_ranges; ################# content-type #################### action beg_mime { guard(base_hd->mime_type); } action set_mime { setguarded(base_hd->mime_type, I); } -action set_charset { +action set_charset { if (buflen < FETCHER_URL_MAX) { buf[buflen++] = 0; base_hd->charset = EncodingHintByName((const char*)buf); @@ -232,12 +232,12 @@ mime_type = "text/plain"i %{c(MIME_TEXT)} ; -charset_name = token_char+ >clear_buf $update_buf; -mime_param = "charset"i ws* '=' ws* '"'? charset_name '"'? %set_charset @2 +charset_name = token_char+ >clear_buf $update_buf; +mime_param = "charset"i ws* '=' ws* '"'? charset_name '"'? %set_charset @2 | token ws* '=' ws* '"'? token '"'? @1 | text $0; mime_parms = (lws ';' lws mime_param)*; -content_type = "content-type"i def %beg_mime mime_type mime_parms eoh %set_mime; +content_type = "content-type"i def %beg_mime mime_type mime_parms eoh %set_mime; ################# last modified ################### action beg_modtime { guard(base_hd->http_time); } @@ -245,10 +245,10 @@ action set_modtime { setguarded(base_hd->http_time, DateTimeFields.ToTimeT(-1)); } -last_modified = "last-modified"i def %beg_modtime http_date eoh %set_modtime; +last_modified = "last-modified"i def %beg_modtime http_date eoh %set_modtime; ################# location ######################## -action set_location { +action set_location { while (buflen > 0 && (buf[buflen - 1] == ' ' || buf[buflen - 1] == '\t')) { buflen --; } @@ -350,16 +350,16 @@ action set_squid_error { squid_error = "X-Yandex-Squid-Error"i def any_text eoh %set_squid_error; ################# auth ######################## -action init_auth { +action init_auth { if (auth_hd) auth_hd->use_auth=true; } -action update_auth_buf +action update_auth_buf { if (auth_hd && buflen < sizeof(buf)) buf[buflen++] = *fpc; } quoted_str = /"/ (text_char - /"/)* /"/ >2; -auth_quoted_str = ( /"/ ( ( text_char - /"/ )* >clear_buf $update_auth_buf ) /"/ ) > 2; +auth_quoted_str = ( /"/ ( ( text_char - /"/ )* >clear_buf $update_auth_buf ) /"/ ) > 2; # do not support auth-int, too heavy procedure @@ -394,7 +394,7 @@ auth_param = auth_good_param @1 | auth_params = auth_param ( ws* /,/ ws* auth_param )*; -digest_challenge = ("digest"i %init_auth ws+ auth_params) | +digest_challenge = ("digest"i %init_auth ws+ auth_params) | ((token-"digest"i) text); auth = "www-authenticate"i def digest_challenge eoh; @@ -597,7 +597,7 @@ request_header = message_header $0 | request_priority @1; ################# main ############################ -action accepted { lastchar = (char*)fpc; return 2; } +action accepted { lastchar = (char*)fpc; return 2; } main := ((response_status_line ('\r'? response_header)*) | (request_status_line ('\r' ? request_header)*)) @@ -628,10 +628,10 @@ machine http_chunk_parser; alphtype unsigned char; -action clear_hex { cnt64 = 0; } -action update_hex { cnt64 = 16 * cnt64 + X(fc); if(cnt64 > Max<int>()) return -2; } -action set_chunk { chunk_length = static_cast<int>(cnt64); } -action accepted { lastchar = (char*)fpc; return 2; } +action clear_hex { cnt64 = 0; } +action update_hex { cnt64 = 16 * cnt64 + X(fc); if(cnt64 > Max<int>()) return -2; } +action set_chunk { chunk_length = static_cast<int>(cnt64); } +action accepted { lastchar = (char*)fpc; return 2; } eol = '\r'? '\n'; ws = [ \t]; @@ -648,7 +648,7 @@ token = token_char+; text = (text_char | lw)*; def = lws ':' lws; -hex = (xdigit+) >clear_hex $update_hex; +hex = (xdigit+) >clear_hex $update_hex; quoted_string = '"' ((text_char - '"') $0 | '\\"' @1)* '"'; chunk_ext_val = token | quoted_string; @@ -658,9 +658,9 @@ chunk_extension = ws* (';' chunk_ext_name ws* '=' ws* chunk_ext_val ws*)*; entity_header = token def text eoh; trailer = entity_header*; -chunk = (hex - '0'+) chunk_extension? %set_chunk; +chunk = (hex - '0'+) chunk_extension? %set_chunk; last_chunk = '0'+ chunk_extension? eol trailer; -main := eol (chunk $0 | last_chunk @1) eol @accepted; +main := eol (chunk $0 | last_chunk @1) eol @accepted; }%% diff --git a/library/cpp/http/fetch/httpfsm_ut.cpp b/library/cpp/http/fetch/httpfsm_ut.cpp index 3320a850d0..b018e80101 100644 --- a/library/cpp/http/fetch/httpfsm_ut.cpp +++ b/library/cpp/http/fetch/httpfsm_ut.cpp @@ -5,7 +5,7 @@ #include <util/generic/ptr.h> #include <library/cpp/charset/doccodes.h> #include <library/cpp/testing/unittest/registar.h> - + class THttpHeaderParserTestSuite: public TTestBase { UNIT_TEST_SUITE(THttpHeaderParserTestSuite); UNIT_TEST(TestRequestHeader); @@ -63,10 +63,10 @@ public: void TestMimeType(); void TestRepeatedContentEncoding(); }; - + void THttpHeaderParserTestSuite::TestStart() { httpHeaderParser.Reset(new THttpHeaderParser()); -} +} void THttpHeaderParserTestSuite::TestFinish() { httpHeaderParser.Reset(); diff --git a/library/cpp/http/fetch/ya.make b/library/cpp/http/fetch/ya.make index 8f2db959d1..7737127463 100644 --- a/library/cpp/http/fetch/ya.make +++ b/library/cpp/http/fetch/ya.make @@ -20,7 +20,7 @@ SRCS( httpheader.cpp httpload.cpp exthttpcodes.cpp - httpfsm.rl6 + httpfsm.rl6 httpagent.h httpfetcher.h httpheader.h diff --git a/library/cpp/http/misc/httpdate.cpp b/library/cpp/http/misc/httpdate.cpp index d0754d2001..4a3031bbf4 100644 --- a/library/cpp/http/misc/httpdate.cpp +++ b/library/cpp/http/misc/httpdate.cpp @@ -36,9 +36,9 @@ #include <ctime> #include <util/system/compat.h> /* stricmp */ -#include <util/system/yassert.h> +#include <util/system/yassert.h> #include "httpdate.h" -#include <util/datetime/base.h> +#include <util/datetime/base.h> static const char *wkdays[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" diff --git a/library/cpp/http/misc/httpdate.h b/library/cpp/http/misc/httpdate.h index 9b839223d5..04876f38fe 100644 --- a/library/cpp/http/misc/httpdate.h +++ b/library/cpp/http/misc/httpdate.h @@ -8,13 +8,13 @@ #define BAD_DATE ((time_t)-1) inline time_t parse_http_date(const TStringBuf& datestring) { - try { + try { return TInstant::ParseHttpDeprecated(datestring).TimeT(); - } catch (const TDateTimeParseException&) { - return BAD_DATE; - } -} - + } catch (const TDateTimeParseException&) { + return BAD_DATE; + } +} + int format_http_date(char buf[], size_t size, time_t when); char* format_http_date(time_t when, char* buf, size_t len); diff --git a/library/cpp/http/misc/httpreqdata.h b/library/cpp/http/misc/httpreqdata.h index a273399859..16e59c4d78 100644 --- a/library/cpp/http/misc/httpreqdata.h +++ b/library/cpp/http/misc/httpreqdata.h @@ -12,7 +12,7 @@ #include <util/generic/hash.h> #include <util/system/yassert.h> #include <util/generic/string.h> -#include <util/datetime/base.h> +#include <util/datetime/base.h> #include <util/generic/buffer.h> using THttpHeadersContainer = THashMap<TString, TString, TCIOps, TCIOps>; diff --git a/library/cpp/http/server/http.cpp b/library/cpp/http/server/http.cpp index 4ef72ad921..128583bdd7 100644 --- a/library/cpp/http/server/http.cpp +++ b/library/cpp/http/server/http.cpp @@ -42,7 +42,7 @@ namespace { struct TShouldStop { }; - + struct TWakeupPollAble: public IPollAble { void OnPollEvent(TInstant) override { throw TShouldStop(); @@ -187,14 +187,14 @@ public: // Start the listener thread ListenerRunningOK = false; - // throws on error - TPipeHandle::Pipe(ListenWakeupReadFd, ListenWakeupWriteFd); - - SetNonBlock(ListenWakeupWriteFd, true); - SetNonBlock(ListenWakeupReadFd, true); - - Poller->WaitRead(ListenWakeupReadFd, &WakeupPollAble); - + // throws on error + TPipeHandle::Pipe(ListenWakeupReadFd, ListenWakeupWriteFd); + + SetNonBlock(ListenWakeupWriteFd, true); + SetNonBlock(ListenWakeupReadFd, true); + + Poller->WaitRead(ListenWakeupReadFd, &WakeupPollAble); + ListenStartEvent.Reset(); try { ListenThread.Reset(new TThread(ListenSocketFunction, this)); @@ -220,8 +220,8 @@ public: } void Stop() { - Shutdown(); - + Shutdown(); + TGuard<TMutex> g(StopMutex); if (ListenThread) { ListenThread->Join(); @@ -238,8 +238,8 @@ public: } void Shutdown() { - ListenWakeupWriteFd.Write("", 1); - // ignore result + ListenWakeupWriteFd.Write("", 1); + // ignore result } void AddRequest(TAutoPtr<TClientRequest> req, bool fail) { @@ -443,8 +443,8 @@ public: } THolder<TThread> ListenThread; - TPipeHandle ListenWakeupReadFd; - TPipeHandle ListenWakeupWriteFd; + TPipeHandle ListenWakeupReadFd; + TPipeHandle ListenWakeupWriteFd; TSystemEvent ListenStartEvent; TMtpQueueRef Requests; TMtpQueueRef FailRequests; diff --git a/library/cpp/lfalloc/ya.make b/library/cpp/lfalloc/ya.make index 0ac987b52e..cace05f9d8 100644 --- a/library/cpp/lfalloc/ya.make +++ b/library/cpp/lfalloc/ya.make @@ -16,10 +16,10 @@ ELSE() ) ENDIF() -PEERDIR( +PEERDIR( library/cpp/malloc/api -) - +) + SET(IDE_FOLDER "util") END() diff --git a/library/cpp/lfalloc/yt/ya.make b/library/cpp/lfalloc/yt/ya.make index ddd657d2ab..8c1a4f8a72 100644 --- a/library/cpp/lfalloc/yt/ya.make +++ b/library/cpp/lfalloc/yt/ya.make @@ -20,10 +20,10 @@ ELSE() ) ENDIF() -PEERDIR( +PEERDIR( library/cpp/malloc/api -) - +) + SET(IDE_FOLDER "util") END() diff --git a/library/cpp/lwtrace/all.h b/library/cpp/lwtrace/all.h index 0b286ceb95..d7aa57c49d 100644 --- a/library/cpp/lwtrace/all.h +++ b/library/cpp/lwtrace/all.h @@ -4,7 +4,7 @@ #include "event.h" #include "preprocessor.h" #include "probe.h" -#include "start.h" +#include "start.h" // // Full documentation: https://wiki.yandex-team.ru/development/poisk/arcadia/library/lwtrace/ diff --git a/library/cpp/lwtrace/kill_action.cpp b/library/cpp/lwtrace/kill_action.cpp index cd3316e90d..2b74dc4587 100644 --- a/library/cpp/lwtrace/kill_action.cpp +++ b/library/cpp/lwtrace/kill_action.cpp @@ -1,21 +1,21 @@ #include "kill_action.h" - -#ifndef _win_ -#include <sys/types.h> -#include <signal.h> -#endif - + +#ifndef _win_ +#include <sys/types.h> +#include <signal.h> +#endif + #include <stdlib.h> - -using namespace NLWTrace; -using namespace NLWTrace::NPrivate; - + +using namespace NLWTrace; +using namespace NLWTrace::NPrivate; + bool TKillActionExecutor::DoExecute(TOrbit&, const TParams&) { -#ifdef _win_ - abort(); -#else - int r = kill(getpid(), SIGABRT); +#ifdef _win_ + abort(); +#else + int r = kill(getpid(), SIGABRT); Y_VERIFY(r == 0, "kill failed"); return true; -#endif -} +#endif +} diff --git a/library/cpp/lwtrace/kill_action.h b/library/cpp/lwtrace/kill_action.h index 16cc6b1a76..14da9ffd50 100644 --- a/library/cpp/lwtrace/kill_action.h +++ b/library/cpp/lwtrace/kill_action.h @@ -1,7 +1,7 @@ -#pragma once - -#include "probe.h" - +#pragma once + +#include "probe.h" + namespace NLWTrace { namespace NPrivate { class TKillActionExecutor: public IExecutor { @@ -10,6 +10,6 @@ namespace NLWTrace { } bool DoExecute(TOrbit& orbit, const TParams& params) override; }; - + } } diff --git a/library/cpp/lwtrace/preprocessor.h b/library/cpp/lwtrace/preprocessor.h index ca6559aac8..40865467b2 100644 --- a/library/cpp/lwtrace/preprocessor.h +++ b/library/cpp/lwtrace/preprocessor.h @@ -3,18 +3,18 @@ #include "check.h" #include "perf.h" #include "symbol.h" - + #include <util/generic/hide_ptr.h> #include <util/system/platform.h> #include <stddef.h> //size_t -#ifdef _win_ -#ifndef LWTRACE_DISABLE -#define LWTRACE_DISABLE -#endif // LWTRACE_DISABLE -#endif // _win_ - +#ifdef _win_ +#ifndef LWTRACE_DISABLE +#define LWTRACE_DISABLE +#endif // LWTRACE_DISABLE +#endif // _win_ + // Maximum number of executors that can be attached to a probe #define LWTRACE_MAX_ACTIONS 100 diff --git a/library/cpp/lwtrace/protos/lwtrace.proto b/library/cpp/lwtrace/protos/lwtrace.proto index 74890b1952..0051095719 100644 --- a/library/cpp/lwtrace/protos/lwtrace.proto +++ b/library/cpp/lwtrace/protos/lwtrace.proto @@ -44,9 +44,9 @@ message TLogAction { uint32 MaxRecords = 4; // Do not write more than MaxRecords records to the log (count from the trace beginning, not start) } -message TPrintToStderrAction { -} - +message TPrintToStderrAction { +} + message TKillAction { } diff --git a/library/cpp/lwtrace/protos/ya.make b/library/cpp/lwtrace/protos/ya.make index 1b9fd06c75..503d5e515f 100644 --- a/library/cpp/lwtrace/protos/ya.make +++ b/library/cpp/lwtrace/protos/ya.make @@ -5,7 +5,7 @@ OWNER(serxa) INCLUDE_TAGS(GO_PROTO) SRCS( - lwtrace.proto + lwtrace.proto ) END() diff --git a/library/cpp/lwtrace/start.cpp b/library/cpp/lwtrace/start.cpp index 1dd231ccea..121d5472b6 100644 --- a/library/cpp/lwtrace/start.cpp +++ b/library/cpp/lwtrace/start.cpp @@ -1,19 +1,19 @@ #include "start.h" - + #include "all.h" #include <google/protobuf/text_format.h> - -#include <util/generic/singleton.h> -#include <util/stream/file.h> + +#include <util/generic/singleton.h> +#include <util/stream/file.h> #include <util/stream/output.h> #include <util/system/env.h> - + #include <stdlib.h> - -using namespace NLWTrace; - -namespace { + +using namespace NLWTrace; + +namespace { struct TTraceManagerHolder { TManager TraceManager; TTraceManagerHolder() @@ -42,16 +42,16 @@ void NLWTrace::StartLwtraceFromEnv() { TString path = GetEnv("LWTRACE"); if (!path) { - return; - } - - try { + return; + } + + try { TraceFromEnv(path); - } catch (...) { - Cerr << "failed to load lwtrace script: " << CurrentExceptionMessage() << "\n"; - abort(); - } -} + } catch (...) { + Cerr << "failed to load lwtrace script: " << CurrentExceptionMessage() << "\n"; + abort(); + } +} void NLWTrace::StartLwtraceFromEnv(std::function<void(TManager&)> prepare) { TString path = GetEnv("LWTRACE"); diff --git a/library/cpp/lwtrace/start.h b/library/cpp/lwtrace/start.h index 4c360f4625..2755212bff 100644 --- a/library/cpp/lwtrace/start.h +++ b/library/cpp/lwtrace/start.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + #include <functional> -namespace NLWTrace { +namespace NLWTrace { class TManager; - + void StartLwtraceFromEnv(); void StartLwtraceFromEnv(std::function<void(TManager&)> prepare); - -} + +} diff --git a/library/cpp/lwtrace/stderr_writer.cpp b/library/cpp/lwtrace/stderr_writer.cpp index 8d635d42f6..6e5654c338 100644 --- a/library/cpp/lwtrace/stderr_writer.cpp +++ b/library/cpp/lwtrace/stderr_writer.cpp @@ -1,19 +1,19 @@ -#include "stderr_writer.h" +#include "stderr_writer.h" #include <util/stream/str.h> - -using namespace NLWTrace; - + +using namespace NLWTrace; + bool TStderrActionExecutor::DoExecute(TOrbit&, const TParams& params) { TString ParamValues[LWTRACE_MAX_PARAMS]; - Probe->Event.Signature.SerializeParams(params, ParamValues); - + Probe->Event.Signature.SerializeParams(params, ParamValues); + TStringStream ss; ss << Probe->Event.GetProvider() << "." << Probe->Event.Name; - for (ui32 i = 0; i < Probe->Event.Signature.ParamCount; ++i) { + for (ui32 i = 0; i < Probe->Event.Signature.ParamCount; ++i) { ss << " " << Probe->Event.Signature.ParamNames[i] << "=" << ParamValues[i]; - } + } ss << "\n"; Cerr << ss.Str(); - return true; -} + return true; +} diff --git a/library/cpp/lwtrace/stderr_writer.h b/library/cpp/lwtrace/stderr_writer.h index b1375cbd27..b17fc3136e 100644 --- a/library/cpp/lwtrace/stderr_writer.h +++ b/library/cpp/lwtrace/stderr_writer.h @@ -1,19 +1,19 @@ -#pragma once - -#include "probe.h" - -namespace NLWTrace { +#pragma once + +#include "probe.h" + +namespace NLWTrace { class TStderrActionExecutor: public IExecutor { private: TProbe* const Probe; - + public: explicit TStderrActionExecutor(TProbe* probe) : Probe(probe) { } - + bool DoExecute(TOrbit& orbit, const TParams& params) override; }; -} +} diff --git a/library/cpp/lwtrace/ya.make b/library/cpp/lwtrace/ya.make index fb7b59629b..d9accb3006 100644 --- a/library/cpp/lwtrace/ya.make +++ b/library/cpp/lwtrace/ya.make @@ -10,7 +10,7 @@ SRCS( check.cpp control.cpp custom_action.cpp - kill_action.cpp + kill_action.cpp log_shuttle.cpp perf.cpp probes.cpp diff --git a/library/cpp/malloc/api/malloc.cpp b/library/cpp/malloc/api/malloc.cpp index 6f646033b7..eed1c58a38 100644 --- a/library/cpp/malloc/api/malloc.cpp +++ b/library/cpp/malloc/api/malloc.cpp @@ -1,8 +1,8 @@ #include <stdlib.h> #include <stdio.h> -#include "malloc.h" - +#include "malloc.h" + namespace { bool SetEmptyParam(const char*, const char*) { return false; diff --git a/library/cpp/malloc/api/malloc.h b/library/cpp/malloc/api/malloc.h index 3185cb3769..ebd545d6dd 100644 --- a/library/cpp/malloc/api/malloc.h +++ b/library/cpp/malloc/api/malloc.h @@ -1,25 +1,25 @@ -#pragma once - +#pragma once + #include <string.h> #include <util/system/compiler.h> -namespace NMalloc { - struct TMallocInfo { - TMallocInfo(); +namespace NMalloc { + struct TMallocInfo { + TMallocInfo(); - const char* Name; + const char* Name; bool (*SetParam)(const char* param, const char* value); const char* (*GetParam)(const char* param); bool (*CheckParam)(const char* param, bool defaultValue); - }; - + }; + extern volatile bool IsAllocatorCorrupted; void AbortFromCorruptedAllocator(const char* errorMessage = nullptr); - // this function should be implemented by malloc implementations - TMallocInfo MallocInfo(); + // this function should be implemented by malloc implementations + TMallocInfo MallocInfo(); struct TAllocHeader { void* Block; @@ -29,4 +29,4 @@ namespace NMalloc { AllocSize = size | signature; } }; -} +} diff --git a/library/cpp/malloc/api/ut/ut.cpp b/library/cpp/malloc/api/ut/ut.cpp index d806b6f637..7eccd0bf8d 100644 --- a/library/cpp/malloc/api/ut/ut.cpp +++ b/library/cpp/malloc/api/ut/ut.cpp @@ -1,10 +1,10 @@ #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/malloc/api/malloc.h> - + Y_UNIT_TEST_SUITE(MallocApi) { Y_UNIT_TEST(ToStream) { - TStringStream ss; - ss << NMalloc::MallocInfo(); - } -} + TStringStream ss; + ss << NMalloc::MallocInfo(); + } +} diff --git a/library/cpp/malloc/api/ut/ya.make b/library/cpp/malloc/api/ut/ya.make index 8ce4ce20cf..e57225b45d 100644 --- a/library/cpp/malloc/api/ut/ya.make +++ b/library/cpp/malloc/api/ut/ya.make @@ -1,13 +1,13 @@ UNITTEST() - -OWNER(nga) - + +OWNER(nga) + PEERDIR( library/cpp/malloc/api/helpers ) -SRCS( - ut.cpp -) - -END() +SRCS( + ut.cpp +) + +END() diff --git a/library/cpp/malloc/api/ya.make b/library/cpp/malloc/api/ya.make index fdf6c294ea..0ebaa0c589 100644 --- a/library/cpp/malloc/api/ya.make +++ b/library/cpp/malloc/api/ya.make @@ -1,11 +1,11 @@ -LIBRARY() - +LIBRARY() + NO_UTIL() - -OWNER(nga) - -SRCS( - malloc.cpp -) - -END() + +OWNER(nga) + +SRCS( + malloc.cpp +) + +END() diff --git a/library/cpp/malloc/jemalloc/malloc-info.cpp b/library/cpp/malloc/jemalloc/malloc-info.cpp index 060cce9fa9..2643ca4766 100644 --- a/library/cpp/malloc/jemalloc/malloc-info.cpp +++ b/library/cpp/malloc/jemalloc/malloc-info.cpp @@ -1,5 +1,5 @@ #include <library/cpp/malloc/api/malloc.h> - + using namespace NMalloc; #if defined(_MSC_VER) @@ -12,7 +12,7 @@ TMallocInfo NMalloc::MallocInfo() { #include <strings.h> #include <stdlib.h> #include <inttypes.h> - + #include <contrib/libs/jemalloc/include/jemalloc/jemalloc.h> namespace { @@ -55,11 +55,11 @@ namespace { } } -TMallocInfo NMalloc::MallocInfo() { - TMallocInfo r; - r.Name = "jemalloc"; +TMallocInfo NMalloc::MallocInfo() { + TMallocInfo r; + r.Name = "jemalloc"; r.SetParam = JESetParam; r.GetParam = JEGetParam; - return r; -} + return r; +} #endif diff --git a/library/cpp/malloc/jemalloc/ya.make b/library/cpp/malloc/jemalloc/ya.make index 0398578d6c..99db474eab 100644 --- a/library/cpp/malloc/jemalloc/ya.make +++ b/library/cpp/malloc/jemalloc/ya.make @@ -1,9 +1,9 @@ -LIBRARY() - +LIBRARY() + NO_UTIL() - -OWNER(nga) - + +OWNER(nga) + IF (OS_ANDROID) PEERDIR( library/cpp/malloc/system @@ -17,5 +17,5 @@ ELSE() malloc-info.cpp ) ENDIF() - -END() + +END() diff --git a/library/cpp/malloc/ya.make b/library/cpp/malloc/ya.make index 045d1e24f2..0ec9db71d2 100644 --- a/library/cpp/malloc/ya.make +++ b/library/cpp/malloc/ya.make @@ -1,4 +1,4 @@ -RECURSE( +RECURSE( api api/helpers api/ut @@ -13,7 +13,7 @@ RECURSE( mimalloc/link_test hu hu/link_test -) +) IF (NOT OS_WINDOWS) RECURSE( diff --git a/library/cpp/messagebus/acceptor.cpp b/library/cpp/messagebus/acceptor.cpp index d54be57fd9..64a38619c2 100644 --- a/library/cpp/messagebus/acceptor.cpp +++ b/library/cpp/messagebus/acceptor.cpp @@ -1,127 +1,127 @@ #include "acceptor.h" - -#include "key_value_printer.h" -#include "mb_lwtrace.h" + +#include "key_value_printer.h" +#include "mb_lwtrace.h" #include "network.h" - + #include <util/network/init.h> #include <util/system/defaults.h> #include <util/system/error.h> #include <util/system/yassert.h> - -LWTRACE_USING(LWTRACE_MESSAGEBUS_PROVIDER) - -using namespace NActor; -using namespace NBus; -using namespace NBus::NPrivate; - -TAcceptor::TAcceptor(TBusSessionImpl* session, ui64 acceptorId, SOCKET socket, const TNetAddr& addr) - : TActor<TAcceptor>(session->Queue->WorkQueue.Get()) - , AcceptorId(acceptorId) - , Session(session) + +LWTRACE_USING(LWTRACE_MESSAGEBUS_PROVIDER) + +using namespace NActor; +using namespace NBus; +using namespace NBus::NPrivate; + +TAcceptor::TAcceptor(TBusSessionImpl* session, ui64 acceptorId, SOCKET socket, const TNetAddr& addr) + : TActor<TAcceptor>(session->Queue->WorkQueue.Get()) + , AcceptorId(acceptorId) + , Session(session) , GranStatus(session->Config.Secret.StatusFlushPeriod) -{ - SetNonBlock(socket, true); - - Channel = Session->ReadEventLoop.Register(socket, this); - Channel->EnableRead(); - - Stats.AcceptorId = acceptorId; - Stats.Fd = socket; - Stats.ListenAddr = addr; - +{ + SetNonBlock(socket, true); + + Channel = Session->ReadEventLoop.Register(socket, this); + Channel->EnableRead(); + + Stats.AcceptorId = acceptorId; + Stats.Fd = socket; + Stats.ListenAddr = addr; + SendStatus(TInstant::Now()); -} - -void TAcceptor::Act(TDefaultTag) { - EShutdownState state = ShutdownState.State.Get(); - - if (state == SS_SHUTDOWN_COMPLETE) { - return; - } - +} + +void TAcceptor::Act(TDefaultTag) { + EShutdownState state = ShutdownState.State.Get(); + + if (state == SS_SHUTDOWN_COMPLETE) { + return; + } + TInstant now = TInstant::Now(); - if (state == SS_SHUTDOWN_COMMAND) { - if (!!Channel) { - Channel->Unregister(); - Channel.Drop(); - Stats.Fd = INVALID_SOCKET; - } - + if (state == SS_SHUTDOWN_COMMAND) { + if (!!Channel) { + Channel->Unregister(); + Channel.Drop(); + Stats.Fd = INVALID_SOCKET; + } + SendStatus(now); - - Session->GetDeadAcceptorStatusQueue()->EnqueueAndSchedule(Stats); - Stats.ResetIncremental(); - - ShutdownState.CompleteShutdown(); - return; - } - - THolder<TOpaqueAddr> addr(new TOpaqueAddr()); - SOCKET acceptedSocket = accept(Channel->GetSocket(), addr->MutableAddr(), addr->LenPtr()); - - int acceptErrno = LastSystemError(); - - if (acceptedSocket == INVALID_SOCKET) { - if (LastSystemError() != EWOULDBLOCK) { - Stats.LastAcceptErrorErrno = acceptErrno; - Stats.LastAcceptErrorInstant = now; - ++Stats.AcceptErrorCount; - } - } else { - TSocketHolder s(acceptedSocket); - try { - SetKeepAlive(s, true); - SetNoDelay(s, Session->Config.TcpNoDelay); - SetSockOptTcpCork(s, Session->Config.TcpCork); - SetCloseOnExec(s, true); - SetNonBlock(s, true); + + Session->GetDeadAcceptorStatusQueue()->EnqueueAndSchedule(Stats); + Stats.ResetIncremental(); + + ShutdownState.CompleteShutdown(); + return; + } + + THolder<TOpaqueAddr> addr(new TOpaqueAddr()); + SOCKET acceptedSocket = accept(Channel->GetSocket(), addr->MutableAddr(), addr->LenPtr()); + + int acceptErrno = LastSystemError(); + + if (acceptedSocket == INVALID_SOCKET) { + if (LastSystemError() != EWOULDBLOCK) { + Stats.LastAcceptErrorErrno = acceptErrno; + Stats.LastAcceptErrorInstant = now; + ++Stats.AcceptErrorCount; + } + } else { + TSocketHolder s(acceptedSocket); + try { + SetKeepAlive(s, true); + SetNoDelay(s, Session->Config.TcpNoDelay); + SetSockOptTcpCork(s, Session->Config.TcpCork); + SetCloseOnExec(s, true); + SetNonBlock(s, true); if (Session->Config.SocketToS >= 0) { SetSocketToS(s, addr.Get(), Session->Config.SocketToS); } - } catch (...) { - // It means that connection was reset just now - // TODO: do something better - goto skipAccept; - } - - { - TOnAccept onAccept; - onAccept.s = s.Release(); - onAccept.addr = TNetAddr(addr.Release()); - onAccept.now = now; - - LWPROBE(Accepted, ToString(onAccept.addr)); - - Session->GetOnAcceptQueue()->EnqueueAndSchedule(onAccept); - + } catch (...) { + // It means that connection was reset just now + // TODO: do something better + goto skipAccept; + } + + { + TOnAccept onAccept; + onAccept.s = s.Release(); + onAccept.addr = TNetAddr(addr.Release()); + onAccept.now = now; + + LWPROBE(Accepted, ToString(onAccept.addr)); + + Session->GetOnAcceptQueue()->EnqueueAndSchedule(onAccept); + Stats.LastAcceptSuccessInstant = now; - ++Stats.AcceptSuccessCount; - } - + ++Stats.AcceptSuccessCount; + } + skipAccept:; - } - - Channel->EnableRead(); - + } + + Channel->EnableRead(); + SendStatus(now); -} - +} + void TAcceptor::SendStatus(TInstant now) { GranStatus.Listen.Update(Stats, now); -} - -void TAcceptor::HandleEvent(SOCKET socket, void* cookie) { +} + +void TAcceptor::HandleEvent(SOCKET socket, void* cookie) { Y_UNUSED(socket); Y_UNUSED(cookie); - - GetActor()->Schedule(); -} - -void TAcceptor::Shutdown() { - ShutdownState.ShutdownCommand(); - GetActor()->Schedule(); - - ShutdownState.ShutdownComplete.WaitI(); -} + + GetActor()->Schedule(); +} + +void TAcceptor::Shutdown() { + ShutdownState.ShutdownCommand(); + GetActor()->Schedule(); + + ShutdownState.ShutdownComplete.WaitI(); +} diff --git a/library/cpp/messagebus/acceptor.h b/library/cpp/messagebus/acceptor.h index 602381839e..57cb010bf2 100644 --- a/library/cpp/messagebus/acceptor.h +++ b/library/cpp/messagebus/acceptor.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include "acceptor_status.h" #include "defs.h" -#include "event_loop.h" -#include "netaddr.h" -#include "session_impl.h" -#include "shutdown_state.h" - +#include "event_loop.h" +#include "netaddr.h" +#include "session_impl.h" +#include "shutdown_state.h" + #include <library/cpp/messagebus/actor/actor.h> #include <util/system/event.h> @@ -18,43 +18,43 @@ namespace NBus { private ::NActor::TActor<TAcceptor> { friend struct TBusSessionImpl; friend class ::NActor::TActor<TAcceptor>; - + public: TAcceptor(TBusSessionImpl* session, ui64 acceptorId, SOCKET socket, const TNetAddr& addr); - + void HandleEvent(SOCKET socket, void* cookie) override; - + void Shutdown(); - + inline ::NActor::TActor<TAcceptor>* GetActor() { return this; } - + private: void SendStatus(TInstant now); void Act(::NActor::TDefaultTag); - + private: const ui64 AcceptorId; - + TBusSessionImpl* const Session; NEventLoop::TChannelPtr Channel; - + TAcceptorStatus Stats; - + TAtomicShutdownState ShutdownState; - + struct TGranStatus { TGranStatus(TDuration gran) : Listen(gran) { } - + TGranUp<TAcceptorStatus> Listen; }; - + TGranStatus GranStatus; - }; + }; } } diff --git a/library/cpp/messagebus/acceptor_status.cpp b/library/cpp/messagebus/acceptor_status.cpp index 3b0725a46b..5006ff68ae 100644 --- a/library/cpp/messagebus/acceptor_status.cpp +++ b/library/cpp/messagebus/acceptor_status.cpp @@ -1,68 +1,68 @@ #include "acceptor_status.h" - -#include "key_value_printer.h" - + +#include "key_value_printer.h" + #include <util/stream/format.h> #include <util/stream/output.h> - -using namespace NBus; -using namespace NBus::NPrivate; - -TAcceptorStatus::TAcceptorStatus() - : Summary(false) - , AcceptorId(0) - , Fd(INVALID_SOCKET) -{ - ResetIncremental(); -} - -void TAcceptorStatus::ResetIncremental() { - AcceptSuccessCount = 0; - AcceptErrorCount = 0; - LastAcceptErrorErrno = 0; - LastAcceptErrorInstant = TInstant(); - LastAcceptSuccessInstant = TInstant(); -} - -TAcceptorStatus& TAcceptorStatus::operator+=(const TAcceptorStatus& that) { + +using namespace NBus; +using namespace NBus::NPrivate; + +TAcceptorStatus::TAcceptorStatus() + : Summary(false) + , AcceptorId(0) + , Fd(INVALID_SOCKET) +{ + ResetIncremental(); +} + +void TAcceptorStatus::ResetIncremental() { + AcceptSuccessCount = 0; + AcceptErrorCount = 0; + LastAcceptErrorErrno = 0; + LastAcceptErrorInstant = TInstant(); + LastAcceptSuccessInstant = TInstant(); +} + +TAcceptorStatus& TAcceptorStatus::operator+=(const TAcceptorStatus& that) { Y_ASSERT(Summary); Y_ASSERT(AcceptorId == 0); - - AcceptSuccessCount += that.AcceptSuccessCount; - LastAcceptSuccessInstant = Max(LastAcceptSuccessInstant, that.LastAcceptSuccessInstant); - - AcceptErrorCount += that.AcceptErrorCount; - if (that.LastAcceptErrorInstant > LastAcceptErrorInstant) { - LastAcceptErrorInstant = that.LastAcceptErrorInstant; - LastAcceptErrorErrno = that.LastAcceptErrorErrno; - } - - return *this; -} - + + AcceptSuccessCount += that.AcceptSuccessCount; + LastAcceptSuccessInstant = Max(LastAcceptSuccessInstant, that.LastAcceptSuccessInstant); + + AcceptErrorCount += that.AcceptErrorCount; + if (that.LastAcceptErrorInstant > LastAcceptErrorInstant) { + LastAcceptErrorInstant = that.LastAcceptErrorInstant; + LastAcceptErrorErrno = that.LastAcceptErrorErrno; + } + + return *this; +} + TString TAcceptorStatus::PrintToString() const { - TStringStream ss; - - if (!Summary) { - ss << "acceptor (" << AcceptorId << "), fd=" << Fd << ", addr=" << ListenAddr << Endl; - } - - TKeyValuePrinter p; - - p.AddRow("accept error count", LeftPad(AcceptErrorCount, 4)); - - if (AcceptErrorCount > 0) { - p.AddRow("last accept error", + TStringStream ss; + + if (!Summary) { + ss << "acceptor (" << AcceptorId << "), fd=" << Fd << ", addr=" << ListenAddr << Endl; + } + + TKeyValuePrinter p; + + p.AddRow("accept error count", LeftPad(AcceptErrorCount, 4)); + + if (AcceptErrorCount > 0) { + p.AddRow("last accept error", TString() + LastSystemErrorText(LastAcceptErrorErrno) + " at " + LastAcceptErrorInstant.ToString()); - } - - p.AddRow("accept success count", LeftPad(AcceptSuccessCount, 4)); - if (AcceptSuccessCount > 0) { - p.AddRow("last accept success", + } + + p.AddRow("accept success count", LeftPad(AcceptSuccessCount, 4)); + if (AcceptSuccessCount > 0) { + p.AddRow("last accept success", TString() + "at " + LastAcceptSuccessInstant.ToString()); - } - - ss << p.PrintToString(); - - return ss.Str(); -} + } + + ss << p.PrintToString(); + + return ss.Str(); +} diff --git a/library/cpp/messagebus/acceptor_status.h b/library/cpp/messagebus/acceptor_status.h index 40be70107d..6aa1404f4d 100644 --- a/library/cpp/messagebus/acceptor_status.h +++ b/library/cpp/messagebus/acceptor_status.h @@ -1,35 +1,35 @@ -#pragma once - +#pragma once + #include "netaddr.h" -#include <util/network/init.h> - +#include <util/network/init.h> + namespace NBus { namespace NPrivate { struct TAcceptorStatus { bool Summary; - + ui64 AcceptorId; - + SOCKET Fd; - + TNetAddr ListenAddr; - + unsigned AcceptSuccessCount; TInstant LastAcceptSuccessInstant; - + unsigned AcceptErrorCount; TInstant LastAcceptErrorInstant; int LastAcceptErrorErrno; - + void ResetIncremental(); - + TAcceptorStatus(); - + TAcceptorStatus& operator+=(const TAcceptorStatus& that); - + TString PrintToString() const; }; - + } } diff --git a/library/cpp/messagebus/actor/actor.h b/library/cpp/messagebus/actor/actor.h index 6c05d474ae..9b8f20298a 100644 --- a/library/cpp/messagebus/actor/actor.h +++ b/library/cpp/messagebus/actor/actor.h @@ -1,100 +1,100 @@ -#pragma once - +#pragma once + #include "executor.h" -#include "tasks.h" -#include "what_thread_does.h" - +#include "tasks.h" +#include "what_thread_does.h" + #include <util/system/yassert.h> -namespace NActor { +namespace NActor { class IActor: protected IWorkItem { public: // TODO: make private TTasks Tasks; - + public: virtual void ScheduleHereV() = 0; virtual void ScheduleV() = 0; virtual void ScheduleHereAtMostOnceV() = 0; - + // TODO: make private virtual void RefV() = 0; virtual void UnRefV() = 0; - + // mute warnings ~IActor() override { } }; - + struct TDefaultTag {}; - + template <typename TThis, typename TTag = TDefaultTag> class TActor: public IActor { private: TExecutor* const Executor; - + public: TActor(TExecutor* executor) : Executor(executor) { } - + void AddTaskFromActorLoop() { bool schedule = Tasks.AddTask(); // TODO: check thread id Y_ASSERT(!schedule); } - + /** - * Schedule actor. - * - * If actor is sleeping, then actor will be executed right now. - * If actor is executing right now, it will be executed one more time. - * If this method is called multiple time, actor will be re-executed no more than one more time. - */ + * Schedule actor. + * + * If actor is sleeping, then actor will be executed right now. + * If actor is executing right now, it will be executed one more time. + * If this method is called multiple time, actor will be re-executed no more than one more time. + */ void Schedule() { if (Tasks.AddTask()) { EnqueueWork(); } - } - + } + /** - * Schedule actor, execute it in current thread. - * - * If actor is running, continue executing where it is executing. - * If actor is sleeping, execute it in current thread. - * - * Operation is useful for tasks that are likely to complete quickly. - */ + * Schedule actor, execute it in current thread. + * + * If actor is running, continue executing where it is executing. + * If actor is sleeping, execute it in current thread. + * + * Operation is useful for tasks that are likely to complete quickly. + */ void ScheduleHere() { if (Tasks.AddTask()) { Loop(); } - } - + } + /** - * Schedule actor, execute in current thread no more than once. - * - * If actor is running, continue executing where it is executing. - * If actor is sleeping, execute one iteration here, and if actor got new tasks, - * reschedule it in worker pool. - */ + * Schedule actor, execute in current thread no more than once. + * + * If actor is running, continue executing where it is executing. + * If actor is sleeping, execute one iteration here, and if actor got new tasks, + * reschedule it in worker pool. + */ void ScheduleHereAtMostOnce() { if (Tasks.AddTask()) { bool fetched = Tasks.FetchTask(); Y_VERIFY(fetched, "happens"); - + DoAct(); - + // if someone added more tasks, schedule them if (Tasks.FetchTask()) { bool added = Tasks.AddTask(); Y_VERIFY(!added, "happens"); EnqueueWork(); } - } - } - + } + } + void ScheduleHereV() override { ScheduleHere(); } @@ -110,35 +110,35 @@ namespace NActor { void UnRefV() override { GetThis()->UnRef(); } - + private: TThis* GetThis() { return static_cast<TThis*>(this); } - + void EnqueueWork() { GetThis()->Ref(); Executor->EnqueueWork({this}); } - + void DoAct() { WHAT_THREAD_DOES_PUSH_POP_CURRENT_FUNC(); - + GetThis()->Act(TTag()); } - + void Loop() { // TODO: limit number of iterations while (Tasks.FetchTask()) { DoAct(); } - } - + } + void DoWork() override { Y_ASSERT(GetThis()->RefCount() >= 1); Loop(); GetThis()->UnRef(); } }; - + } diff --git a/library/cpp/messagebus/actor/actor_ut.cpp b/library/cpp/messagebus/actor/actor_ut.cpp index 3f6d72ccdc..b76ab55bfa 100644 --- a/library/cpp/messagebus/actor/actor_ut.cpp +++ b/library/cpp/messagebus/actor/actor_ut.cpp @@ -1,157 +1,157 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "actor.h" -#include "queue_in_actor.h" - + +#include "actor.h" +#include "queue_in_actor.h" + #include <library/cpp/messagebus/misc/test_sync.h> #include <util/generic/object_counter.h> #include <util/system/event.h> -using namespace NActor; - -template <typename TThis> +using namespace NActor; + +template <typename TThis> struct TTestActorBase: public TAtomicRefCount<TThis>, public TActor<TThis> { - TTestSync Started; - TTestSync Acted; - - TTestActorBase(TExecutor* executor) - : TActor<TThis>(executor) + TTestSync Started; + TTestSync Acted; + + TTestActorBase(TExecutor* executor) + : TActor<TThis>(executor) { } - - void Act(TDefaultTag) { - Started.Inc(); - static_cast<TThis*>(this)->Act2(); - Acted.Inc(); - } -}; - + + void Act(TDefaultTag) { + Started.Inc(); + static_cast<TThis*>(this)->Act2(); + Acted.Inc(); + } +}; + struct TNopActor: public TTestActorBase<TNopActor> { - TObjectCounter<TNopActor> AllocCounter; - - TNopActor(TExecutor* executor) - : TTestActorBase<TNopActor>(executor) + TObjectCounter<TNopActor> AllocCounter; + + TNopActor(TExecutor* executor) + : TTestActorBase<TNopActor>(executor) { } - - void Act2() { - } -}; - + + void Act2() { + } +}; + struct TWaitForSignalActor: public TTestActorBase<TWaitForSignalActor> { - TWaitForSignalActor(TExecutor* executor) - : TTestActorBase<TWaitForSignalActor>(executor) + TWaitForSignalActor(TExecutor* executor) + : TTestActorBase<TWaitForSignalActor>(executor) { } - + TSystemEvent WaitFor; - - void Act2() { - WaitFor.Wait(); - } -}; - + + void Act2() { + WaitFor.Wait(); + } +}; + struct TDecrementAndSendActor: public TTestActorBase<TDecrementAndSendActor>, public TQueueInActor<TDecrementAndSendActor, int> { TSystemEvent Done; - - TDecrementAndSendActor* Next; - - TDecrementAndSendActor(TExecutor* executor) - : TTestActorBase<TDecrementAndSendActor>(executor) + + TDecrementAndSendActor* Next; + + TDecrementAndSendActor(TExecutor* executor) + : TTestActorBase<TDecrementAndSendActor>(executor) , Next(nullptr) { } - - void ProcessItem(TDefaultTag, TDefaultTag, int n) { - if (n == 0) { - Done.Signal(); - } else { - Next->EnqueueAndSchedule(n - 1); - } - } - - void Act(TDefaultTag) { - DequeueAll(); - } -}; - -struct TObjectCountChecker { - TObjectCountChecker() { - CheckCounts(); - } - - ~TObjectCountChecker() { - CheckCounts(); - } - - void CheckCounts() { - UNIT_ASSERT_VALUES_EQUAL(TAtomicBase(0), TObjectCounter<TNopActor>::ObjectCount()); - UNIT_ASSERT_VALUES_EQUAL(TAtomicBase(0), TObjectCounter<TWaitForSignalActor>::ObjectCount()); - UNIT_ASSERT_VALUES_EQUAL(TAtomicBase(0), TObjectCounter<TDecrementAndSendActor>::ObjectCount()); - } -}; - + + void ProcessItem(TDefaultTag, TDefaultTag, int n) { + if (n == 0) { + Done.Signal(); + } else { + Next->EnqueueAndSchedule(n - 1); + } + } + + void Act(TDefaultTag) { + DequeueAll(); + } +}; + +struct TObjectCountChecker { + TObjectCountChecker() { + CheckCounts(); + } + + ~TObjectCountChecker() { + CheckCounts(); + } + + void CheckCounts() { + UNIT_ASSERT_VALUES_EQUAL(TAtomicBase(0), TObjectCounter<TNopActor>::ObjectCount()); + UNIT_ASSERT_VALUES_EQUAL(TAtomicBase(0), TObjectCounter<TWaitForSignalActor>::ObjectCount()); + UNIT_ASSERT_VALUES_EQUAL(TAtomicBase(0), TObjectCounter<TDecrementAndSendActor>::ObjectCount()); + } +}; + Y_UNIT_TEST_SUITE(TActor) { Y_UNIT_TEST(Simple) { - TObjectCountChecker objectCountChecker; - - TExecutor executor(4); - - TIntrusivePtr<TNopActor> actor(new TNopActor(&executor)); - - actor->Schedule(); - - actor->Acted.WaitFor(1u); - } - + TObjectCountChecker objectCountChecker; + + TExecutor executor(4); + + TIntrusivePtr<TNopActor> actor(new TNopActor(&executor)); + + actor->Schedule(); + + actor->Acted.WaitFor(1u); + } + Y_UNIT_TEST(ScheduleAfterStart) { - TObjectCountChecker objectCountChecker; - - TExecutor executor(4); - - TIntrusivePtr<TWaitForSignalActor> actor(new TWaitForSignalActor(&executor)); - - actor->Schedule(); - - actor->Started.WaitFor(1); - - actor->Schedule(); - - actor->WaitFor.Signal(); - - // make sure Act is called second time - actor->Acted.WaitFor(2u); - } - - void ComplexImpl(int queueSize, int actorCount) { - TObjectCountChecker objectCountChecker; - - TExecutor executor(queueSize); - + TObjectCountChecker objectCountChecker; + + TExecutor executor(4); + + TIntrusivePtr<TWaitForSignalActor> actor(new TWaitForSignalActor(&executor)); + + actor->Schedule(); + + actor->Started.WaitFor(1); + + actor->Schedule(); + + actor->WaitFor.Signal(); + + // make sure Act is called second time + actor->Acted.WaitFor(2u); + } + + void ComplexImpl(int queueSize, int actorCount) { + TObjectCountChecker objectCountChecker; + + TExecutor executor(queueSize); + TVector<TIntrusivePtr<TDecrementAndSendActor>> actors; - for (int i = 0; i < actorCount; ++i) { - actors.push_back(new TDecrementAndSendActor(&executor)); - } - - for (int i = 0; i < actorCount; ++i) { - actors.at(i)->Next = &*actors.at((i + 1) % actorCount); - } - - for (int i = 0; i < actorCount; ++i) { - actors.at(i)->EnqueueAndSchedule(10000); - } - - for (int i = 0; i < actorCount; ++i) { - actors.at(i)->Done.WaitI(); - } - } - + for (int i = 0; i < actorCount; ++i) { + actors.push_back(new TDecrementAndSendActor(&executor)); + } + + for (int i = 0; i < actorCount; ++i) { + actors.at(i)->Next = &*actors.at((i + 1) % actorCount); + } + + for (int i = 0; i < actorCount; ++i) { + actors.at(i)->EnqueueAndSchedule(10000); + } + + for (int i = 0; i < actorCount; ++i) { + actors.at(i)->Done.WaitI(); + } + } + Y_UNIT_TEST(ComplexContention) { - ComplexImpl(4, 6); - } - + ComplexImpl(4, 6); + } + Y_UNIT_TEST(ComplexNoContention) { - ComplexImpl(6, 4); - } -} + ComplexImpl(6, 4); + } +} diff --git a/library/cpp/messagebus/actor/executor.cpp b/library/cpp/messagebus/actor/executor.cpp index fbd76a86ba..7a2227a458 100644 --- a/library/cpp/messagebus/actor/executor.cpp +++ b/library/cpp/messagebus/actor/executor.cpp @@ -1,95 +1,95 @@ #include "executor.h" - -#include "thread_extra.h" -#include "what_thread_does.h" -#include "what_thread_does_guard.h" - + +#include "thread_extra.h" +#include "what_thread_does.h" +#include "what_thread_does_guard.h" + #include <util/generic/utility.h> #include <util/random/random.h> #include <util/stream/str.h> #include <util/system/tls.h> #include <util/system/yassert.h> - + #include <array> -using namespace NActor; -using namespace NActor::NPrivate; - -namespace { - struct THistoryInternal { - struct TRecord { +using namespace NActor; +using namespace NActor::NPrivate; + +namespace { + struct THistoryInternal { + struct TRecord { TAtomic MaxQueueSize; - + TRecord() : MaxQueueSize() { } - - TExecutorHistory::THistoryRecord Capture() { - TExecutorHistory::THistoryRecord r; + + TExecutorHistory::THistoryRecord Capture() { + TExecutorHistory::THistoryRecord r; r.MaxQueueSize = AtomicGet(MaxQueueSize); - return r; - } - }; - - ui64 Start; - ui64 LastTime; - + return r; + } + }; + + ui64 Start; + ui64 LastTime; + std::array<TRecord, 3600> Records; - - THistoryInternal() { - Start = TInstant::Now().Seconds(); - LastTime = Start - 1; - } - - TRecord& GetRecordForTime(ui64 time) { - return Records[time % Records.size()]; - } - - TRecord& GetNowRecord(ui64 now) { - for (ui64 t = LastTime + 1; t <= now; ++t) { - GetRecordForTime(t) = TRecord(); - } - LastTime = now; - return GetRecordForTime(now); - } - - TExecutorHistory Capture() { - TExecutorHistory history; - ui64 now = TInstant::Now().Seconds(); - ui64 lastHistoryRecord = now - 1; - ui32 historySize = Min<ui32>(lastHistoryRecord - Start, Records.size() - 1); - history.HistoryRecords.resize(historySize); - for (ui32 i = 0; i < historySize; ++i) { - history.HistoryRecords[i] = GetRecordForTime(lastHistoryRecord - historySize + i).Capture(); - } - history.LastHistoryRecordSecond = lastHistoryRecord; - return history; - } - }; - -} - + + THistoryInternal() { + Start = TInstant::Now().Seconds(); + LastTime = Start - 1; + } + + TRecord& GetRecordForTime(ui64 time) { + return Records[time % Records.size()]; + } + + TRecord& GetNowRecord(ui64 now) { + for (ui64 t = LastTime + 1; t <= now; ++t) { + GetRecordForTime(t) = TRecord(); + } + LastTime = now; + return GetRecordForTime(now); + } + + TExecutorHistory Capture() { + TExecutorHistory history; + ui64 now = TInstant::Now().Seconds(); + ui64 lastHistoryRecord = now - 1; + ui32 historySize = Min<ui32>(lastHistoryRecord - Start, Records.size() - 1); + history.HistoryRecords.resize(historySize); + for (ui32 i = 0; i < historySize; ++i) { + history.HistoryRecords[i] = GetRecordForTime(lastHistoryRecord - historySize + i).Capture(); + } + history.LastHistoryRecordSecond = lastHistoryRecord; + return history; + } + }; + +} + Y_POD_STATIC_THREAD(TExecutor*) ThreadCurrentExecutor; - -static const char* NoLocation = "nowhere"; - -struct TExecutorWorkerThreadLocalData { - ui32 MaxQueueSize; -}; - -static TExecutorWorkerThreadLocalData WorkerNoThreadLocalData; + +static const char* NoLocation = "nowhere"; + +struct TExecutorWorkerThreadLocalData { + ui32 MaxQueueSize; +}; + +static TExecutorWorkerThreadLocalData WorkerNoThreadLocalData; Y_POD_STATIC_THREAD(TExecutorWorkerThreadLocalData) WorkerThreadLocalData; - -namespace NActor { + +namespace NActor { struct TExecutorWorker { TExecutor* const Executor; TThread Thread; const char** WhatThreadDoesLocation; TExecutorWorkerThreadLocalData* ThreadLocalData; - + TExecutorWorker(TExecutor* executor) : Executor(executor) , Thread(RunThreadProc, this) @@ -98,241 +98,241 @@ namespace NActor { { Thread.Start(); } - + void Run() { WhatThreadDoesLocation = ::WhatThreadDoesLocation(); AtomicSet(ThreadLocalData, &::WorkerThreadLocalData); WHAT_THREAD_DOES_PUSH_POP_CURRENT_FUNC(); Executor->RunWorker(); } - + static void* RunThreadProc(void* thiz0) { TExecutorWorker* thiz = (TExecutorWorker*)thiz0; thiz->Run(); return nullptr; } }; - + struct TExecutor::TImpl { TExecutor* const Executor; THistoryInternal History; - + TSystemEvent HelperStopSignal; TThread HelperThread; - + TImpl(TExecutor* executor) : Executor(executor) , HelperThread(HelperThreadProc, this) { } - + void RunHelper() { ui64 nowSeconds = TInstant::Now().Seconds(); for (;;) { TInstant nextStop = TInstant::Seconds(nowSeconds + 1) + TDuration::MilliSeconds(RandomNumber<ui32>(1000)); - + if (HelperStopSignal.WaitD(nextStop)) { return; } - + nowSeconds = nextStop.Seconds(); - + THistoryInternal::TRecord& record = History.GetNowRecord(nowSeconds); - + ui32 maxQueueSize = Executor->GetMaxQueueSizeAndClear(); if (maxQueueSize > record.MaxQueueSize) { AtomicSet(record.MaxQueueSize, maxQueueSize); } - } - } - + } + } + static void* HelperThreadProc(void* impl0) { TImpl* impl = (TImpl*)impl0; impl->RunHelper(); return nullptr; } }; - + +} + +static TExecutor::TConfig MakeConfig(unsigned workerCount) { + TExecutor::TConfig config; + config.WorkerCount = workerCount; + return config; +} + +TExecutor::TExecutor(size_t workerCount) + : Config(MakeConfig(workerCount)) +{ + Init(); +} + +TExecutor::TExecutor(const TExecutor::TConfig& config) + : Config(config) +{ + Init(); } - -static TExecutor::TConfig MakeConfig(unsigned workerCount) { - TExecutor::TConfig config; - config.WorkerCount = workerCount; - return config; -} - -TExecutor::TExecutor(size_t workerCount) - : Config(MakeConfig(workerCount)) -{ - Init(); -} - -TExecutor::TExecutor(const TExecutor::TConfig& config) - : Config(config) -{ - Init(); -} - -void TExecutor::Init() { - Impl.Reset(new TImpl(this)); - + +void TExecutor::Init() { + Impl.Reset(new TImpl(this)); + AtomicSet(ExitWorkers, 0); - + Y_VERIFY(Config.WorkerCount > 0); - - for (size_t i = 0; i < Config.WorkerCount; i++) { - WorkerThreads.push_back(new TExecutorWorker(this)); - } - - Impl->HelperThread.Start(); -} - -TExecutor::~TExecutor() { - Stop(); -} - -void TExecutor::Stop() { + + for (size_t i = 0; i < Config.WorkerCount; i++) { + WorkerThreads.push_back(new TExecutorWorker(this)); + } + + Impl->HelperThread.Start(); +} + +TExecutor::~TExecutor() { + Stop(); +} + +void TExecutor::Stop() { AtomicSet(ExitWorkers, 1); - - Impl->HelperStopSignal.Signal(); - Impl->HelperThread.Join(); - - { - TWhatThreadDoesAcquireGuard<TMutex> guard(WorkMutex, "executor: acquiring lock for Stop"); - WorkAvailable.BroadCast(); - } - - for (size_t i = 0; i < WorkerThreads.size(); i++) { - WorkerThreads[i]->Thread.Join(); - } - - // TODO: make queue empty at this point - ProcessWorkQueueHere(); -} - + + Impl->HelperStopSignal.Signal(); + Impl->HelperThread.Join(); + + { + TWhatThreadDoesAcquireGuard<TMutex> guard(WorkMutex, "executor: acquiring lock for Stop"); + WorkAvailable.BroadCast(); + } + + for (size_t i = 0; i < WorkerThreads.size(); i++) { + WorkerThreads[i]->Thread.Join(); + } + + // TODO: make queue empty at this point + ProcessWorkQueueHere(); +} + void TExecutor::EnqueueWork(TArrayRef<IWorkItem* const> wis) { - if (wis.empty()) - return; - + if (wis.empty()) + return; + if (Y_UNLIKELY(AtomicGet(ExitWorkers) != 0)) { Y_VERIFY(WorkItems.Empty(), "executor %s: cannot add tasks after queue shutdown", Config.Name); - } - - TWhatThreadDoesPushPop pp("executor: EnqueueWork"); - - WorkItems.PushAll(wis); - - { - if (wis.size() == 1) { - TWhatThreadDoesAcquireGuard<TMutex> g(WorkMutex, "executor: acquiring lock for EnqueueWork"); - WorkAvailable.Signal(); - } else { - TWhatThreadDoesAcquireGuard<TMutex> g(WorkMutex, "executor: acquiring lock for EnqueueWork"); - WorkAvailable.BroadCast(); - } - } -} - + } + + TWhatThreadDoesPushPop pp("executor: EnqueueWork"); + + WorkItems.PushAll(wis); + + { + if (wis.size() == 1) { + TWhatThreadDoesAcquireGuard<TMutex> g(WorkMutex, "executor: acquiring lock for EnqueueWork"); + WorkAvailable.Signal(); + } else { + TWhatThreadDoesAcquireGuard<TMutex> g(WorkMutex, "executor: acquiring lock for EnqueueWork"); + WorkAvailable.BroadCast(); + } + } +} + size_t TExecutor::GetWorkQueueSize() const { - return WorkItems.Size(); -} - + return WorkItems.Size(); +} + using namespace NTSAN; -ui32 TExecutor::GetMaxQueueSizeAndClear() const { - ui32 max = 0; - for (unsigned i = 0; i < WorkerThreads.size(); ++i) { +ui32 TExecutor::GetMaxQueueSizeAndClear() const { + ui32 max = 0; + for (unsigned i = 0; i < WorkerThreads.size(); ++i) { TExecutorWorkerThreadLocalData* wtls = RelaxedLoad(&WorkerThreads[i]->ThreadLocalData); max = Max<ui32>(max, RelaxedLoad(&wtls->MaxQueueSize)); RelaxedStore<ui32>(&wtls->MaxQueueSize, 0); - } - return max; -} - + } + return max; +} + TString TExecutor::GetStatus() const { - return GetStatusRecordInternal().Status; -} - + return GetStatusRecordInternal().Status; +} + TString TExecutor::GetStatusSingleLine() const { - TStringStream ss; - ss << "work items: " << GetWorkQueueSize(); - return ss.Str(); -} - + TStringStream ss; + ss << "work items: " << GetWorkQueueSize(); + return ss.Str(); +} + TExecutorStatus TExecutor::GetStatusRecordInternal() const { - TExecutorStatus r; - - r.WorkQueueSize = GetWorkQueueSize(); - - { - TStringStream ss; - ss << "work items: " << GetWorkQueueSize() << "\n"; - ss << "workers:\n"; - for (unsigned i = 0; i < WorkerThreads.size(); ++i) { + TExecutorStatus r; + + r.WorkQueueSize = GetWorkQueueSize(); + + { + TStringStream ss; + ss << "work items: " << GetWorkQueueSize() << "\n"; + ss << "workers:\n"; + for (unsigned i = 0; i < WorkerThreads.size(); ++i) { ss << "-- " << AtomicGet(*AtomicGet(WorkerThreads[i]->WhatThreadDoesLocation)) << "\n"; - } - r.Status = ss.Str(); - } - - r.History = Impl->History.Capture(); - - return r; -} - + } + r.Status = ss.Str(); + } + + r.History = Impl->History.Capture(); + + return r; +} + bool TExecutor::IsInExecutorThread() const { - return ThreadCurrentExecutor == this; -} - -TAutoPtr<IWorkItem> TExecutor::DequeueWork() { - IWorkItem* wi = reinterpret_cast<IWorkItem*>(1); - size_t queueSize = Max<size_t>(); - if (!WorkItems.TryPop(&wi, &queueSize)) { - TWhatThreadDoesAcquireGuard<TMutex> g(WorkMutex, "executor: acquiring lock for DequeueWork"); - while (!WorkItems.TryPop(&wi, &queueSize)) { + return ThreadCurrentExecutor == this; +} + +TAutoPtr<IWorkItem> TExecutor::DequeueWork() { + IWorkItem* wi = reinterpret_cast<IWorkItem*>(1); + size_t queueSize = Max<size_t>(); + if (!WorkItems.TryPop(&wi, &queueSize)) { + TWhatThreadDoesAcquireGuard<TMutex> g(WorkMutex, "executor: acquiring lock for DequeueWork"); + while (!WorkItems.TryPop(&wi, &queueSize)) { if (AtomicGet(ExitWorkers) != 0) return nullptr; - - TWhatThreadDoesPushPop pp("waiting for work on condvar"); - WorkAvailable.Wait(WorkMutex); - } - } + + TWhatThreadDoesPushPop pp("waiting for work on condvar"); + WorkAvailable.Wait(WorkMutex); + } + } auto& wtls = TlsRef(WorkerThreadLocalData); if (queueSize > RelaxedLoad(&wtls.MaxQueueSize)) { RelaxedStore<ui32>(&wtls.MaxQueueSize, queueSize); - } - - return wi; -} - -void TExecutor::RunWorkItem(TAutoPtr<IWorkItem> wi) { - WHAT_THREAD_DOES_PUSH_POP_CURRENT_FUNC(); - wi.Release()->DoWork(); -} - -void TExecutor::ProcessWorkQueueHere() { - IWorkItem* wi; - while (WorkItems.TryPop(&wi)) { - RunWorkItem(wi); - } -} - -void TExecutor::RunWorker() { + } + + return wi; +} + +void TExecutor::RunWorkItem(TAutoPtr<IWorkItem> wi) { + WHAT_THREAD_DOES_PUSH_POP_CURRENT_FUNC(); + wi.Release()->DoWork(); +} + +void TExecutor::ProcessWorkQueueHere() { + IWorkItem* wi; + while (WorkItems.TryPop(&wi)) { + RunWorkItem(wi); + } +} + +void TExecutor::RunWorker() { Y_VERIFY(!ThreadCurrentExecutor, "state check"); - ThreadCurrentExecutor = this; - + ThreadCurrentExecutor = this; + SetCurrentThreadName("wrkr"); - - for (;;) { - TAutoPtr<IWorkItem> wi = DequeueWork(); - if (!wi) { - break; - } - // Note for messagebus users: make sure program crashes - // on uncaught exception in thread, otherewise messagebus may just hang on error. - RunWorkItem(wi); - } - + + for (;;) { + TAutoPtr<IWorkItem> wi = DequeueWork(); + if (!wi) { + break; + } + // Note for messagebus users: make sure program crashes + // on uncaught exception in thread, otherewise messagebus may just hang on error. + RunWorkItem(wi); + } + ThreadCurrentExecutor = (TExecutor*)nullptr; -} +} diff --git a/library/cpp/messagebus/actor/executor.h b/library/cpp/messagebus/actor/executor.h index 2a30580ff1..7292d8be53 100644 --- a/library/cpp/messagebus/actor/executor.h +++ b/library/cpp/messagebus/actor/executor.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include "ring_buffer_with_spin_lock.h" #include <util/generic/array_ref.h> -#include <util/generic/vector.h> -#include <util/system/condvar.h> -#include <util/system/event.h> +#include <util/generic/vector.h> +#include <util/system/condvar.h> +#include <util/system/event.h> #include <util/system/mutex.h> #include <util/system/thread.h> -#include <util/thread/lfqueue.h> - -namespace NActor { +#include <util/thread/lfqueue.h> + +namespace NActor { namespace NPrivate { struct TExecutorHistory { struct THistoryRecord { @@ -18,88 +18,88 @@ namespace NActor { }; TVector<THistoryRecord> HistoryRecords; ui64 LastHistoryRecordSecond; - + ui64 FirstHistoryRecordSecond() const { return LastHistoryRecordSecond - HistoryRecords.size() + 1; } }; - + struct TExecutorStatus { size_t WorkQueueSize = 0; TExecutorHistory History; TString Status; - }; + }; } - + class IWorkItem { public: virtual ~IWorkItem() { - } + } virtual void DoWork(/* must release this */) = 0; - }; - + }; + struct TExecutorWorker; - + class TExecutor: public TAtomicRefCount<TExecutor> { friend struct TExecutorWorker; - + public: struct TConfig { size_t WorkerCount; const char* Name; - + TConfig() : WorkerCount(1) , Name() { } }; - + private: struct TImpl; THolder<TImpl> Impl; - + const TConfig Config; - + TAtomic ExitWorkers; - + TVector<TAutoPtr<TExecutorWorker>> WorkerThreads; - + TRingBufferWithSpinLock<IWorkItem*> WorkItems; - + TMutex WorkMutex; TCondVar WorkAvailable; - + public: explicit TExecutor(size_t workerCount); TExecutor(const TConfig& config); ~TExecutor(); - + void Stop(); - + void EnqueueWork(TArrayRef<IWorkItem* const> w); - + size_t GetWorkQueueSize() const; TString GetStatus() const; TString GetStatusSingleLine() const; NPrivate::TExecutorStatus GetStatusRecordInternal() const; - + bool IsInExecutorThread() const; - + private: void Init(); - + TAutoPtr<IWorkItem> DequeueWork(); - + void ProcessWorkQueueHere(); - + inline void RunWorkItem(TAutoPtr<IWorkItem>); - + void RunWorker(); - + ui32 GetMaxQueueSizeAndClear() const; }; - + using TExecutorPtr = TIntrusivePtr<TExecutor>; - + } diff --git a/library/cpp/messagebus/actor/queue_for_actor.h b/library/cpp/messagebus/actor/queue_for_actor.h index b64a2a27a2..40fa536b82 100644 --- a/library/cpp/messagebus/actor/queue_for_actor.h +++ b/library/cpp/messagebus/actor/queue_for_actor.h @@ -1,65 +1,65 @@ -#pragma once - -#include <util/generic/vector.h> -#include <util/system/yassert.h> -#include <util/thread/lfstack.h> -#include <util/thread/singleton.h> - -// TODO: include from correct directory -#include "temp_tls_vector.h" - -namespace NActor { +#pragma once + +#include <util/generic/vector.h> +#include <util/system/yassert.h> +#include <util/thread/lfstack.h> +#include <util/thread/singleton.h> + +// TODO: include from correct directory +#include "temp_tls_vector.h" + +namespace NActor { namespace NPrivate { struct TTagForTl {}; - + } - + template <typename T> class TQueueForActor { private: TLockFreeStack<T> Queue; - + public: ~TQueueForActor() { Y_VERIFY(Queue.IsEmpty()); } - + bool IsEmpty() { return Queue.IsEmpty(); } - + void Enqueue(const T& value) { Queue.Enqueue(value); } - + template <typename TCollection> void EnqueueAll(const TCollection& all) { Queue.EnqueueAll(all); } - + void Clear() { TVector<T> tmp; Queue.DequeueAll(&tmp); } - + template <typename TFunc> void DequeueAll(const TFunc& func // TODO: , std::enable_if_t<TFunctionParamCount<TFunc>::Value == 1>* = 0 ) { TTempTlsVector<T> temp; - + Queue.DequeueAllSingleConsumer(temp.GetVector()); - + for (typename TVector<T>::reverse_iterator i = temp.GetVector()->rbegin(); i != temp.GetVector()->rend(); ++i) { func(*i); } - + temp.Clear(); - + if (temp.Capacity() * sizeof(T) > 64 * 1024) { temp.Shrink(); } - } + } template <typename TFunc> void DequeueAllLikelyEmpty(const TFunc& func) { @@ -70,5 +70,5 @@ namespace NActor { DequeueAll(func); } }; - + } diff --git a/library/cpp/messagebus/actor/queue_in_actor.h b/library/cpp/messagebus/actor/queue_in_actor.h index f93eb03070..9865996532 100644 --- a/library/cpp/messagebus/actor/queue_in_actor.h +++ b/library/cpp/messagebus/actor/queue_in_actor.h @@ -1,80 +1,80 @@ -#pragma once - -#include "actor.h" -#include "queue_for_actor.h" - +#pragma once + +#include "actor.h" +#include "queue_for_actor.h" + #include <functional> -namespace NActor { +namespace NActor { template <typename TItem> class IQueueInActor { public: virtual void EnqueueAndScheduleV(const TItem& item) = 0; virtual void DequeueAllV() = 0; virtual void DequeueAllLikelyEmptyV() = 0; - + virtual ~IQueueInActor() { } }; - + template <typename TThis, typename TItem, typename TActorTag = TDefaultTag, typename TQueueTag = TDefaultTag> class TQueueInActor: public IQueueInActor<TItem> { typedef TQueueInActor<TThis, TItem, TActorTag, TQueueTag> TSelf; - + public: // TODO: make protected TQueueForActor<TItem> QueueInActor; - + private: TActor<TThis, TActorTag>* GetActor() { return GetThis(); } - + TThis* GetThis() { return static_cast<TThis*>(this); } - + void ProcessItem(const TItem& item) { GetThis()->ProcessItem(TActorTag(), TQueueTag(), item); } - + public: void EnqueueAndNoSchedule(const TItem& item) { QueueInActor.Enqueue(item); } - + void EnqueueAndSchedule(const TItem& item) { EnqueueAndNoSchedule(item); GetActor()->Schedule(); } - + void EnqueueAndScheduleV(const TItem& item) override { EnqueueAndSchedule(item); } - + void Clear() { QueueInActor.Clear(); } - + void DequeueAll() { QueueInActor.DequeueAll(std::bind(&TSelf::ProcessItem, this, std::placeholders::_1)); } - + void DequeueAllV() override { return DequeueAll(); } - + void DequeueAllLikelyEmpty() { QueueInActor.DequeueAllLikelyEmpty(std::bind(&TSelf::ProcessItem, this, std::placeholders::_1)); } - + void DequeueAllLikelyEmptyV() override { return DequeueAllLikelyEmpty(); } - + bool IsEmpty() { return QueueInActor.IsEmpty(); } }; - + } diff --git a/library/cpp/messagebus/actor/ring_buffer.h b/library/cpp/messagebus/actor/ring_buffer.h index a0e2f481bf..ec5706f7c7 100644 --- a/library/cpp/messagebus/actor/ring_buffer.h +++ b/library/cpp/messagebus/actor/ring_buffer.h @@ -1,135 +1,135 @@ -#pragma once - +#pragma once + #include <util/generic/array_ref.h> #include <util/generic/maybe.h> #include <util/generic/utility.h> #include <util/generic/vector.h> -#include <util/system/yassert.h> - -template <typename T> -struct TRingBuffer { -private: - ui32 CapacityPow; - ui32 CapacityMask; - ui32 Capacity; - ui32 WritePos; - ui32 ReadPos; +#include <util/system/yassert.h> + +template <typename T> +struct TRingBuffer { +private: + ui32 CapacityPow; + ui32 CapacityMask; + ui32 Capacity; + ui32 WritePos; + ui32 ReadPos; TVector<T> Data; - - void StateCheck() const { + + void StateCheck() const { Y_ASSERT(Capacity == Data.size()); Y_ASSERT(Capacity == (1u << CapacityPow)); Y_ASSERT((Capacity & CapacityMask) == 0u); Y_ASSERT(Capacity - CapacityMask == 1u); Y_ASSERT(WritePos < Capacity); Y_ASSERT(ReadPos < Capacity); - } - - size_t Writable() const { - return (Capacity + ReadPos - WritePos - 1) & CapacityMask; - } - - void ReserveWritable(ui32 sz) { - if (sz <= Writable()) - return; - - ui32 newCapacityPow = CapacityPow; + } + + size_t Writable() const { + return (Capacity + ReadPos - WritePos - 1) & CapacityMask; + } + + void ReserveWritable(ui32 sz) { + if (sz <= Writable()) + return; + + ui32 newCapacityPow = CapacityPow; while ((1u << newCapacityPow) < sz + ui32(Size()) + 1u) { - ++newCapacityPow; - } - ui32 newCapacity = 1u << newCapacityPow; - ui32 newCapacityMask = newCapacity - 1u; + ++newCapacityPow; + } + ui32 newCapacity = 1u << newCapacityPow; + ui32 newCapacityMask = newCapacity - 1u; TVector<T> newData(newCapacity); - ui32 oldSize = Size(); - // Copy old elements - for (size_t i = 0; i < oldSize; ++i) { - newData[i] = Get(i); - } - - CapacityPow = newCapacityPow; - Capacity = newCapacity; - CapacityMask = newCapacityMask; - Data.swap(newData); - ReadPos = 0; - WritePos = oldSize; - - StateCheck(); - } - - const T& Get(ui32 i) const { - return Data[(ReadPos + i) & CapacityMask]; - } - -public: - TRingBuffer() - : CapacityPow(0) - , CapacityMask(0) - , Capacity(1 << CapacityPow) - , WritePos(0) - , ReadPos(0) - , Data(Capacity) - { - StateCheck(); - } - - size_t Size() const { - return (Capacity + WritePos - ReadPos) & CapacityMask; - } - - bool Empty() const { - return WritePos == ReadPos; - } - + ui32 oldSize = Size(); + // Copy old elements + for (size_t i = 0; i < oldSize; ++i) { + newData[i] = Get(i); + } + + CapacityPow = newCapacityPow; + Capacity = newCapacity; + CapacityMask = newCapacityMask; + Data.swap(newData); + ReadPos = 0; + WritePos = oldSize; + + StateCheck(); + } + + const T& Get(ui32 i) const { + return Data[(ReadPos + i) & CapacityMask]; + } + +public: + TRingBuffer() + : CapacityPow(0) + , CapacityMask(0) + , Capacity(1 << CapacityPow) + , WritePos(0) + , ReadPos(0) + , Data(Capacity) + { + StateCheck(); + } + + size_t Size() const { + return (Capacity + WritePos - ReadPos) & CapacityMask; + } + + bool Empty() const { + return WritePos == ReadPos; + } + void PushAll(TArrayRef<const T> value) { - ReserveWritable(value.size()); - - ui32 secondSize; - ui32 firstSize; - - if (WritePos + value.size() <= Capacity) { - firstSize = value.size(); - secondSize = 0; - } else { - firstSize = Capacity - WritePos; - secondSize = value.size() - firstSize; - } - - for (size_t i = 0; i < firstSize; ++i) { + ReserveWritable(value.size()); + + ui32 secondSize; + ui32 firstSize; + + if (WritePos + value.size() <= Capacity) { + firstSize = value.size(); + secondSize = 0; + } else { + firstSize = Capacity - WritePos; + secondSize = value.size() - firstSize; + } + + for (size_t i = 0; i < firstSize; ++i) { Data[WritePos + i] = value[i]; - } - - for (size_t i = 0; i < secondSize; ++i) { + } + + for (size_t i = 0; i < secondSize; ++i) { Data[i] = value[firstSize + i]; - } - - WritePos = (WritePos + value.size()) & CapacityMask; - StateCheck(); - } - - void Push(const T& t) { + } + + WritePos = (WritePos + value.size()) & CapacityMask; + StateCheck(); + } + + void Push(const T& t) { PushAll(MakeArrayRef(&t, 1)); - } - - bool TryPop(T* r) { - StateCheck(); - if (Empty()) { - return false; - } - *r = Data[ReadPos]; - ReadPos = (ReadPos + 1) & CapacityMask; - return true; - } - - TMaybe<T> TryPop() { - T tmp; - if (TryPop(&tmp)) { - return tmp; - } else { - return TMaybe<T>(); - } - } - - T Pop() { - return *TryPop(); - } -}; + } + + bool TryPop(T* r) { + StateCheck(); + if (Empty()) { + return false; + } + *r = Data[ReadPos]; + ReadPos = (ReadPos + 1) & CapacityMask; + return true; + } + + TMaybe<T> TryPop() { + T tmp; + if (TryPop(&tmp)) { + return tmp; + } else { + return TMaybe<T>(); + } + } + + T Pop() { + return *TryPop(); + } +}; diff --git a/library/cpp/messagebus/actor/ring_buffer_ut.cpp b/library/cpp/messagebus/actor/ring_buffer_ut.cpp index 9b2f46957b..bdb379b3a9 100644 --- a/library/cpp/messagebus/actor/ring_buffer_ut.cpp +++ b/library/cpp/messagebus/actor/ring_buffer_ut.cpp @@ -1,60 +1,60 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "ring_buffer.h" - + +#include "ring_buffer.h" + #include <util/random/random.h> Y_UNIT_TEST_SUITE(RingBuffer) { - struct TRingBufferTester { - TRingBuffer<unsigned> RingBuffer; - - unsigned NextPush; - unsigned NextPop; - + struct TRingBufferTester { + TRingBuffer<unsigned> RingBuffer; + + unsigned NextPush; + unsigned NextPop; + TRingBufferTester() : NextPush() , NextPop() { } - - void Push() { - //Cerr << "push " << NextPush << "\n"; - RingBuffer.Push(NextPush); - NextPush += 1; - } - - void Pop() { - //Cerr << "pop " << NextPop << "\n"; - unsigned popped = RingBuffer.Pop(); - UNIT_ASSERT_VALUES_EQUAL(NextPop, popped); - NextPop += 1; - } - - bool Empty() const { - UNIT_ASSERT_VALUES_EQUAL(RingBuffer.Size(), NextPush - NextPop); - UNIT_ASSERT_VALUES_EQUAL(RingBuffer.Empty(), RingBuffer.Size() == 0); - return RingBuffer.Empty(); - } - }; - - void Iter() { - TRingBufferTester rb; - - while (rb.NextPush < 1000) { - rb.Push(); - while (!rb.Empty() && RandomNumber<bool>()) { - rb.Pop(); - } - } - - while (!rb.Empty()) { - rb.Pop(); - } - } - + + void Push() { + //Cerr << "push " << NextPush << "\n"; + RingBuffer.Push(NextPush); + NextPush += 1; + } + + void Pop() { + //Cerr << "pop " << NextPop << "\n"; + unsigned popped = RingBuffer.Pop(); + UNIT_ASSERT_VALUES_EQUAL(NextPop, popped); + NextPop += 1; + } + + bool Empty() const { + UNIT_ASSERT_VALUES_EQUAL(RingBuffer.Size(), NextPush - NextPop); + UNIT_ASSERT_VALUES_EQUAL(RingBuffer.Empty(), RingBuffer.Size() == 0); + return RingBuffer.Empty(); + } + }; + + void Iter() { + TRingBufferTester rb; + + while (rb.NextPush < 1000) { + rb.Push(); + while (!rb.Empty() && RandomNumber<bool>()) { + rb.Pop(); + } + } + + while (!rb.Empty()) { + rb.Pop(); + } + } + Y_UNIT_TEST(Random) { - for (unsigned i = 0; i < 100; ++i) { - Iter(); - } - } -} + for (unsigned i = 0; i < 100; ++i) { + Iter(); + } + } +} diff --git a/library/cpp/messagebus/actor/ring_buffer_with_spin_lock.h b/library/cpp/messagebus/actor/ring_buffer_with_spin_lock.h index d05dec8577..f0b7cd90e4 100644 --- a/library/cpp/messagebus/actor/ring_buffer_with_spin_lock.h +++ b/library/cpp/messagebus/actor/ring_buffer_with_spin_lock.h @@ -1,91 +1,91 @@ -#pragma once - +#pragma once + #include "ring_buffer.h" -#include <util/system/spinlock.h> - -template <typename T> -class TRingBufferWithSpinLock { -private: - TRingBuffer<T> RingBuffer; - TSpinLock SpinLock; +#include <util/system/spinlock.h> + +template <typename T> +class TRingBufferWithSpinLock { +private: + TRingBuffer<T> RingBuffer; + TSpinLock SpinLock; TAtomic CachedSize; -public: - TRingBufferWithSpinLock() +public: + TRingBufferWithSpinLock() : CachedSize(0) { } - - void Push(const T& t) { - PushAll(t); - } - + + void Push(const T& t) { + PushAll(t); + } + void PushAll(TArrayRef<const T> collection) { - if (collection.empty()) { - return; - } - - TGuard<TSpinLock> Guard(SpinLock); - RingBuffer.PushAll(collection); + if (collection.empty()) { + return; + } + + TGuard<TSpinLock> Guard(SpinLock); + RingBuffer.PushAll(collection); AtomicSet(CachedSize, RingBuffer.Size()); - } - + } + bool TryPop(T* r, size_t* sizePtr = nullptr) { if (AtomicGet(CachedSize) == 0) { - return false; - } - - bool ok; - size_t size; - { - TGuard<TSpinLock> Guard(SpinLock); - ok = RingBuffer.TryPop(r); - size = RingBuffer.Size(); + return false; + } + + bool ok; + size_t size; + { + TGuard<TSpinLock> Guard(SpinLock); + ok = RingBuffer.TryPop(r); + size = RingBuffer.Size(); AtomicSet(CachedSize, size); - } - if (!!sizePtr) { - *sizePtr = size; - } - return ok; - } - - TMaybe<T> TryPop() { - T tmp; - if (TryPop(&tmp)) { - return tmp; - } else { - return TMaybe<T>(); - } - } - + } + if (!!sizePtr) { + *sizePtr = size; + } + return ok; + } + + TMaybe<T> TryPop() { + T tmp; + if (TryPop(&tmp)) { + return tmp; + } else { + return TMaybe<T>(); + } + } + bool PushAllAndTryPop(TArrayRef<const T> collection, T* r) { - if (collection.size() == 0) { - return TryPop(r); - } else { + if (collection.size() == 0) { + return TryPop(r); + } else { if (AtomicGet(CachedSize) == 0) { - *r = collection[0]; - if (collection.size() > 1) { - TGuard<TSpinLock> guard(SpinLock); + *r = collection[0]; + if (collection.size() > 1) { + TGuard<TSpinLock> guard(SpinLock); RingBuffer.PushAll(MakeArrayRef(collection.data() + 1, collection.size() - 1)); AtomicSet(CachedSize, RingBuffer.Size()); - } - } else { - TGuard<TSpinLock> guard(SpinLock); - RingBuffer.PushAll(collection); - *r = RingBuffer.Pop(); + } + } else { + TGuard<TSpinLock> guard(SpinLock); + RingBuffer.PushAll(collection); + *r = RingBuffer.Pop(); AtomicSet(CachedSize, RingBuffer.Size()); - } - return true; - } - } - - bool Empty() const { + } + return true; + } + } + + bool Empty() const { return AtomicGet(CachedSize) == 0; - } - - size_t Size() const { - TGuard<TSpinLock> Guard(SpinLock); - return RingBuffer.Size(); - } -}; + } + + size_t Size() const { + TGuard<TSpinLock> Guard(SpinLock); + return RingBuffer.Size(); + } +}; diff --git a/library/cpp/messagebus/actor/tasks.h b/library/cpp/messagebus/actor/tasks.h index 3eab200b38..31d35931d2 100644 --- a/library/cpp/messagebus/actor/tasks.h +++ b/library/cpp/messagebus/actor/tasks.h @@ -1,9 +1,9 @@ -#pragma once - +#pragma once + #include <util/system/atomic.h> -#include <util/system/yassert.h> - -namespace NActor { +#include <util/system/yassert.h> + +namespace NActor { class TTasks { enum { // order of values is important @@ -11,27 +11,27 @@ namespace NActor { E_RUNNING_NO_TASKS, E_RUNNING_GOT_TASKS, }; - + private: TAtomic State; - + public: TTasks() : State(E_WAITING) { - } - + } + // @return true iff caller have to either schedule task or execute it bool AddTask() { // High contention case optimization: AtomicGet is cheaper than AtomicSwap. if (E_RUNNING_GOT_TASKS == AtomicGet(State)) { return false; } - + TAtomicBase oldState = AtomicSwap(&State, E_RUNNING_GOT_TASKS); return oldState == E_WAITING; - } - + } + // called by executor // @return true iff we have to recheck queues bool FetchTask() { @@ -44,5 +44,5 @@ namespace NActor { Y_FAIL("unknown"); } }; - + } diff --git a/library/cpp/messagebus/actor/tasks_ut.cpp b/library/cpp/messagebus/actor/tasks_ut.cpp index 8468109a7d..d80e8451a5 100644 --- a/library/cpp/messagebus/actor/tasks_ut.cpp +++ b/library/cpp/messagebus/actor/tasks_ut.cpp @@ -1,37 +1,37 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "tasks.h" - -using namespace NActor; - + +#include "tasks.h" + +using namespace NActor; + Y_UNIT_TEST_SUITE(TTasks) { Y_UNIT_TEST(AddTask_FetchTask_Simple) { - TTasks tasks; - - UNIT_ASSERT(tasks.AddTask()); - UNIT_ASSERT(!tasks.AddTask()); - UNIT_ASSERT(!tasks.AddTask()); - - UNIT_ASSERT(tasks.FetchTask()); - UNIT_ASSERT(!tasks.FetchTask()); - - UNIT_ASSERT(tasks.AddTask()); - } - + TTasks tasks; + + UNIT_ASSERT(tasks.AddTask()); + UNIT_ASSERT(!tasks.AddTask()); + UNIT_ASSERT(!tasks.AddTask()); + + UNIT_ASSERT(tasks.FetchTask()); + UNIT_ASSERT(!tasks.FetchTask()); + + UNIT_ASSERT(tasks.AddTask()); + } + Y_UNIT_TEST(AddTask_FetchTask_AddTask) { - TTasks tasks; - - UNIT_ASSERT(tasks.AddTask()); - UNIT_ASSERT(!tasks.AddTask()); - - UNIT_ASSERT(tasks.FetchTask()); - UNIT_ASSERT(!tasks.AddTask()); - UNIT_ASSERT(tasks.FetchTask()); - UNIT_ASSERT(!tasks.AddTask()); - UNIT_ASSERT(!tasks.AddTask()); - UNIT_ASSERT(tasks.FetchTask()); - UNIT_ASSERT(!tasks.FetchTask()); - - UNIT_ASSERT(tasks.AddTask()); - } -} + TTasks tasks; + + UNIT_ASSERT(tasks.AddTask()); + UNIT_ASSERT(!tasks.AddTask()); + + UNIT_ASSERT(tasks.FetchTask()); + UNIT_ASSERT(!tasks.AddTask()); + UNIT_ASSERT(tasks.FetchTask()); + UNIT_ASSERT(!tasks.AddTask()); + UNIT_ASSERT(!tasks.AddTask()); + UNIT_ASSERT(tasks.FetchTask()); + UNIT_ASSERT(!tasks.FetchTask()); + + UNIT_ASSERT(tasks.AddTask()); + } +} diff --git a/library/cpp/messagebus/actor/temp_tls_vector.h b/library/cpp/messagebus/actor/temp_tls_vector.h index 5c535dd07c..675d92f5b0 100644 --- a/library/cpp/messagebus/actor/temp_tls_vector.h +++ b/library/cpp/messagebus/actor/temp_tls_vector.h @@ -1,34 +1,34 @@ -#pragma once - +#pragma once + #include "thread_extra.h" #include <util/generic/vector.h> -#include <util/system/yassert.h> - +#include <util/system/yassert.h> + template <typename T, typename TTag = void, template <typename, class> class TVectorType = TVector> -class TTempTlsVector { -private: - struct TTagForTls {}; - +class TTempTlsVector { +private: + struct TTagForTls {}; + TVectorType<T, std::allocator<T>>* Vector; -public: +public: TVectorType<T, std::allocator<T>>* GetVector() { - return Vector; - } - - TTempTlsVector() { + return Vector; + } + + TTempTlsVector() { Vector = FastTlsSingletonWithTag<TVectorType<T, std::allocator<T>>, TTagForTls>(); Y_ASSERT(Vector->empty()); - } - - ~TTempTlsVector() { + } + + ~TTempTlsVector() { Clear(); } void Clear() { - Vector->clear(); - } + Vector->clear(); + } size_t Capacity() const noexcept { return Vector->capacity(); @@ -37,4 +37,4 @@ public: void Shrink() { Vector->shrink_to_fit(); } -}; +}; diff --git a/library/cpp/messagebus/actor/thread_extra.cpp b/library/cpp/messagebus/actor/thread_extra.cpp index 6472dd92f4..048480f255 100644 --- a/library/cpp/messagebus/actor/thread_extra.cpp +++ b/library/cpp/messagebus/actor/thread_extra.cpp @@ -2,29 +2,29 @@ #include <util/stream/str.h> #include <util/system/execpath.h> -#include <util/system/platform.h> -#include <util/system/thread.h> - -namespace { +#include <util/system/platform.h> +#include <util/system/thread.h> + +namespace { #ifdef _linux_ TString GetExecName() { TString execPath = GetExecPath(); - size_t lastSlash = execPath.find_last_of('/'); + size_t lastSlash = execPath.find_last_of('/'); if (lastSlash == TString::npos) { - return execPath; - } else { - return execPath.substr(lastSlash + 1); - } - } + return execPath; + } else { + return execPath.substr(lastSlash + 1); + } + } #endif -} - +} + void SetCurrentThreadName(const char* name) { -#ifdef _linux_ - TStringStream linuxName; - linuxName << GetExecName() << "." << name; +#ifdef _linux_ + TStringStream linuxName; + linuxName << GetExecName() << "." << name; TThread::SetCurrentThreadName(linuxName.Str().data()); -#else +#else TThread::SetCurrentThreadName(name); -#endif -} +#endif +} diff --git a/library/cpp/messagebus/actor/thread_extra.h b/library/cpp/messagebus/actor/thread_extra.h index 002b2d8d5f..b5aa151618 100644 --- a/library/cpp/messagebus/actor/thread_extra.h +++ b/library/cpp/messagebus/actor/thread_extra.h @@ -1,7 +1,7 @@ -#pragma once - -#include <util/thread/singleton.h> - +#pragma once + +#include <util/thread/singleton.h> + namespace NTSAN { template <typename T> inline void RelaxedStore(volatile T* a, T x) { @@ -25,7 +25,7 @@ namespace NTSAN { } void SetCurrentThreadName(const char* name); - + namespace NThreadExtra { namespace NPrivate { template <typename TValue, typename TTag> @@ -34,8 +34,8 @@ namespace NThreadExtra { }; } } - -template <typename TValue, typename TTag> -static inline TValue* FastTlsSingletonWithTag() { + +template <typename TValue, typename TTag> +static inline TValue* FastTlsSingletonWithTag() { return &FastTlsSingleton< ::NThreadExtra::NPrivate::TValueHolder<TValue, TTag>>()->Value; -} +} diff --git a/library/cpp/messagebus/actor/what_thread_does.cpp b/library/cpp/messagebus/actor/what_thread_does.cpp index 94e0c0f64f..bebb6a888c 100644 --- a/library/cpp/messagebus/actor/what_thread_does.cpp +++ b/library/cpp/messagebus/actor/what_thread_does.cpp @@ -1,22 +1,22 @@ #include "what_thread_does.h" - + #include "thread_extra.h" - + #include <util/system/tls.h> Y_POD_STATIC_THREAD(const char*) WhatThreadDoes; - + const char* PushWhatThreadDoes(const char* what) { const char* r = NTSAN::RelaxedLoad(&WhatThreadDoes); NTSAN::RelaxedStore(&WhatThreadDoes, what); - return r; -} - + return r; +} + void PopWhatThreadDoes(const char* prev) { NTSAN::RelaxedStore(&WhatThreadDoes, prev); -} - +} + const char** WhatThreadDoesLocation() { - return &WhatThreadDoes; -} + return &WhatThreadDoes; +} diff --git a/library/cpp/messagebus/actor/what_thread_does.h b/library/cpp/messagebus/actor/what_thread_does.h index 325528fc55..235d2c3700 100644 --- a/library/cpp/messagebus/actor/what_thread_does.h +++ b/library/cpp/messagebus/actor/what_thread_does.h @@ -1,28 +1,28 @@ -#pragma once - -const char* PushWhatThreadDoes(const char* what); -void PopWhatThreadDoes(const char* prev); -const char** WhatThreadDoesLocation(); - -struct TWhatThreadDoesPushPop { -private: - const char* Prev; - -public: - TWhatThreadDoesPushPop(const char* what) { - Prev = PushWhatThreadDoes(what); - } - - ~TWhatThreadDoesPushPop() { - PopWhatThreadDoes(Prev); - } -}; - -#ifdef __GNUC__ -#define WHAT_THREAD_DOES_FUNCTION __PRETTY_FUNCTION__ -#else -#define WHAT_THREAD_DOES_FUNCTION __FUNCTION__ -#endif - -#define WHAT_THREAD_DOES_PUSH_POP_CURRENT_FUNC() \ - TWhatThreadDoesPushPop whatThreadDoesPushPopCurrentFunc(WHAT_THREAD_DOES_FUNCTION) +#pragma once + +const char* PushWhatThreadDoes(const char* what); +void PopWhatThreadDoes(const char* prev); +const char** WhatThreadDoesLocation(); + +struct TWhatThreadDoesPushPop { +private: + const char* Prev; + +public: + TWhatThreadDoesPushPop(const char* what) { + Prev = PushWhatThreadDoes(what); + } + + ~TWhatThreadDoesPushPop() { + PopWhatThreadDoes(Prev); + } +}; + +#ifdef __GNUC__ +#define WHAT_THREAD_DOES_FUNCTION __PRETTY_FUNCTION__ +#else +#define WHAT_THREAD_DOES_FUNCTION __FUNCTION__ +#endif + +#define WHAT_THREAD_DOES_PUSH_POP_CURRENT_FUNC() \ + TWhatThreadDoesPushPop whatThreadDoesPushPopCurrentFunc(WHAT_THREAD_DOES_FUNCTION) diff --git a/library/cpp/messagebus/actor/what_thread_does_guard.h b/library/cpp/messagebus/actor/what_thread_does_guard.h index f0888f0a8d..f104e9e173 100644 --- a/library/cpp/messagebus/actor/what_thread_does_guard.h +++ b/library/cpp/messagebus/actor/what_thread_does_guard.h @@ -1,40 +1,40 @@ -#pragma once - -#include "what_thread_does.h" - -template <class T> -class TWhatThreadDoesAcquireGuard: public TNonCopyable { +#pragma once + +#include "what_thread_does.h" + +template <class T> +class TWhatThreadDoesAcquireGuard: public TNonCopyable { public: inline TWhatThreadDoesAcquireGuard(const T& t, const char* acquire) noexcept { Init(&t, acquire); } - + inline TWhatThreadDoesAcquireGuard(const T* t, const char* acquire) noexcept { Init(t, acquire); } - + inline ~TWhatThreadDoesAcquireGuard() { Release(); } - + inline void Release() noexcept { if (WasAcquired()) { const_cast<T*>(T_)->Release(); T_ = nullptr; - } + } } - + inline bool WasAcquired() const noexcept { return T_ != nullptr; } - + private: inline void Init(const T* t, const char* acquire) noexcept { T_ = const_cast<T*>(t); TWhatThreadDoesPushPop pp(acquire); T_->Acquire(); } - + private: T* T_; -}; +}; diff --git a/library/cpp/messagebus/actor/what_thread_does_guard_ut.cpp b/library/cpp/messagebus/actor/what_thread_does_guard_ut.cpp index 74137f8f90..e4b218a7ca 100644 --- a/library/cpp/messagebus/actor/what_thread_does_guard_ut.cpp +++ b/library/cpp/messagebus/actor/what_thread_does_guard_ut.cpp @@ -1,13 +1,13 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "what_thread_does_guard.h" - + +#include "what_thread_does_guard.h" + #include <util/system/mutex.h> Y_UNIT_TEST_SUITE(WhatThreadDoesGuard) { Y_UNIT_TEST(Simple) { - TMutex mutex; - - TWhatThreadDoesAcquireGuard<TMutex> guard(mutex, "acquiring my mutex"); - } -} + TMutex mutex; + + TWhatThreadDoesAcquireGuard<TMutex> guard(mutex, "acquiring my mutex"); + } +} diff --git a/library/cpp/messagebus/actor/ya.make b/library/cpp/messagebus/actor/ya.make index 1ea37f5b48..59bd1b0b99 100644 --- a/library/cpp/messagebus/actor/ya.make +++ b/library/cpp/messagebus/actor/ya.make @@ -1,11 +1,11 @@ -LIBRARY(messagebus_actor) - +LIBRARY(messagebus_actor) + OWNER(g:messagebus) - -SRCS( - executor.cpp - thread_extra.cpp - what_thread_does.cpp -) - -END() + +SRCS( + executor.cpp + thread_extra.cpp + what_thread_does.cpp +) + +END() diff --git a/library/cpp/messagebus/all.lwt b/library/cpp/messagebus/all.lwt index f37103a238..0f04be4b2c 100644 --- a/library/cpp/messagebus/all.lwt +++ b/library/cpp/messagebus/all.lwt @@ -1,8 +1,8 @@ -Blocks { - ProbeDesc { - Group: "MessagebusRare" - } - Action { - PrintToStderrAction {} - } -} +Blocks { + ProbeDesc { + Group: "MessagebusRare" + } + Action { + PrintToStderrAction {} + } +} diff --git a/library/cpp/messagebus/all/ya.make b/library/cpp/messagebus/all/ya.make index 3810cf36e5..ffa2dbfabc 100644 --- a/library/cpp/messagebus/all/ya.make +++ b/library/cpp/messagebus/all/ya.make @@ -1,5 +1,5 @@ OWNER(g:messagebus) - + RECURSE_ROOT_RELATIVE( library/python/messagebus library/cpp/messagebus/debug_receiver @@ -7,4 +7,4 @@ RECURSE_ROOT_RELATIVE( library/cpp/messagebus/rain_check library/cpp/messagebus/test library/cpp/messagebus/www -) +) diff --git a/library/cpp/messagebus/async_result.h b/library/cpp/messagebus/async_result.h index 05a1a70f16..d24dde284a 100644 --- a/library/cpp/messagebus/async_result.h +++ b/library/cpp/messagebus/async_result.h @@ -1,54 +1,54 @@ -#pragma once - -#include <util/generic/maybe.h> -#include <util/generic/noncopyable.h> +#pragma once + +#include <util/generic/maybe.h> +#include <util/generic/noncopyable.h> #include <util/system/condvar.h> #include <util/system/mutex.h> #include <util/system/yassert.h> #include <functional> - -// probably this thing should have been called TFuture -template <typename T> -class TAsyncResult : TNonCopyable { -private: - TMutex Mutex; - TCondVar CondVar; - - TMaybe<T> Result; - - typedef void TOnResult(const T&); - + +// probably this thing should have been called TFuture +template <typename T> +class TAsyncResult : TNonCopyable { +private: + TMutex Mutex; + TCondVar CondVar; + + TMaybe<T> Result; + + typedef void TOnResult(const T&); + std::function<TOnResult> OnResult; - -public: - void SetResult(const T& result) { - TGuard<TMutex> guard(Mutex); + +public: + void SetResult(const T& result) { + TGuard<TMutex> guard(Mutex); Y_VERIFY(!Result, "cannot set result twice"); - Result = result; - CondVar.BroadCast(); - - if (!!OnResult) { - OnResult(result); - } - } - - const T& GetResult() { - TGuard<TMutex> guard(Mutex); - while (!Result) { - CondVar.Wait(Mutex); - } - return *Result; - } - - template <typename TFunc> + Result = result; + CondVar.BroadCast(); + + if (!!OnResult) { + OnResult(result); + } + } + + const T& GetResult() { + TGuard<TMutex> guard(Mutex); + while (!Result) { + CondVar.Wait(Mutex); + } + return *Result; + } + + template <typename TFunc> void AndThen(const TFunc& onResult) { - TGuard<TMutex> guard(Mutex); - if (!!Result) { - onResult(*Result); - } else { + TGuard<TMutex> guard(Mutex); + if (!!Result) { + onResult(*Result); + } else { Y_ASSERT(!OnResult); OnResult = std::function<TOnResult>(onResult); - } - } -}; + } + } +}; diff --git a/library/cpp/messagebus/async_result_ut.cpp b/library/cpp/messagebus/async_result_ut.cpp index 93f010d1f5..2e96492afd 100644 --- a/library/cpp/messagebus/async_result_ut.cpp +++ b/library/cpp/messagebus/async_result_ut.cpp @@ -1,37 +1,37 @@ - + #include <library/cpp/testing/unittest/registar.h> - -#include "async_result.h" - -namespace { - void SetValue(int* location, const int& value) { - *location = value; - } - -} - + +#include "async_result.h" + +namespace { + void SetValue(int* location, const int& value) { + *location = value; + } + +} + Y_UNIT_TEST_SUITE(TAsyncResult) { Y_UNIT_TEST(AndThen_Here) { - TAsyncResult<int> r; - - int var = 1; - - r.SetResult(17); - + TAsyncResult<int> r; + + int var = 1; + + r.SetResult(17); + r.AndThen(std::bind(&SetValue, &var, std::placeholders::_1)); - - UNIT_ASSERT_VALUES_EQUAL(17, var); - } - + + UNIT_ASSERT_VALUES_EQUAL(17, var); + } + Y_UNIT_TEST(AndThen_Later) { - TAsyncResult<int> r; - - int var = 1; - + TAsyncResult<int> r; + + int var = 1; + r.AndThen(std::bind(&SetValue, &var, std::placeholders::_1)); - - r.SetResult(17); - - UNIT_ASSERT_VALUES_EQUAL(17, var); - } -} + + r.SetResult(17); + + UNIT_ASSERT_VALUES_EQUAL(17, var); + } +} diff --git a/library/cpp/messagebus/base.h b/library/cpp/messagebus/base.h index adb4ea33b6..79fccc312e 100644 --- a/library/cpp/messagebus/base.h +++ b/library/cpp/messagebus/base.h @@ -1,11 +1,11 @@ #pragma once - + #include <util/system/defaults.h> -namespace NBus { +namespace NBus { /// millis since epoch using TBusInstant = ui64; /// returns time in milliseconds TBusInstant Now(); - -} + +} diff --git a/library/cpp/messagebus/cc_semaphore.h b/library/cpp/messagebus/cc_semaphore.h index 17d3b3bf32..0df8a3d664 100644 --- a/library/cpp/messagebus/cc_semaphore.h +++ b/library/cpp/messagebus/cc_semaphore.h @@ -1,36 +1,36 @@ -#pragma once - -#include "latch.h" - -template <typename TThis> -class TComplexConditionSemaphore { -private: - TLatch Latch; - -public: - void Updated() { - if (GetThis()->TryWait()) { - Latch.Unlock(); - } - } - - void Wait() { - while (!GetThis()->TryWait()) { - Latch.Lock(); - if (GetThis()->TryWait()) { - Latch.Unlock(); - return; - } - Latch.Wait(); - } - } - +#pragma once + +#include "latch.h" + +template <typename TThis> +class TComplexConditionSemaphore { +private: + TLatch Latch; + +public: + void Updated() { + if (GetThis()->TryWait()) { + Latch.Unlock(); + } + } + + void Wait() { + while (!GetThis()->TryWait()) { + Latch.Lock(); + if (GetThis()->TryWait()) { + Latch.Unlock(); + return; + } + Latch.Wait(); + } + } + bool IsLocked() { return Latch.IsLocked(); } - -private: - TThis* GetThis() { - return static_cast<TThis*>(this); - } -}; + +private: + TThis* GetThis() { + return static_cast<TThis*>(this); + } +}; diff --git a/library/cpp/messagebus/cc_semaphore_ut.cpp b/library/cpp/messagebus/cc_semaphore_ut.cpp index 5359ce2f7f..206bb7c96a 100644 --- a/library/cpp/messagebus/cc_semaphore_ut.cpp +++ b/library/cpp/messagebus/cc_semaphore_ut.cpp @@ -1,45 +1,45 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "cc_semaphore.h" - + +#include "cc_semaphore.h" + #include <util/system/atomic.h> -namespace { +namespace { struct TTestSemaphore: public TComplexConditionSemaphore<TTestSemaphore> { - TAtomic Current; - + TAtomic Current; + TTestSemaphore() : Current(0) { } - - bool TryWait() { - return AtomicGet(Current) > 0; - } - - void Aquire() { - Wait(); - AtomicDecrement(Current); - } - - void Release() { - AtomicIncrement(Current); - Updated(); - } - }; -} - + + bool TryWait() { + return AtomicGet(Current) > 0; + } + + void Aquire() { + Wait(); + AtomicDecrement(Current); + } + + void Release() { + AtomicIncrement(Current); + Updated(); + } + }; +} + Y_UNIT_TEST_SUITE(TComplexConditionSemaphore) { Y_UNIT_TEST(Simple) { - TTestSemaphore sema; - UNIT_ASSERT(!sema.TryWait()); - sema.Release(); - UNIT_ASSERT(sema.TryWait()); - sema.Release(); - UNIT_ASSERT(sema.TryWait()); - sema.Aquire(); - UNIT_ASSERT(sema.TryWait()); - sema.Aquire(); - UNIT_ASSERT(!sema.TryWait()); - } -} + TTestSemaphore sema; + UNIT_ASSERT(!sema.TryWait()); + sema.Release(); + UNIT_ASSERT(sema.TryWait()); + sema.Release(); + UNIT_ASSERT(sema.TryWait()); + sema.Aquire(); + UNIT_ASSERT(sema.TryWait()); + sema.Aquire(); + UNIT_ASSERT(!sema.TryWait()); + } +} diff --git a/library/cpp/messagebus/codegen.h b/library/cpp/messagebus/codegen.h index 678d42eb44..83e969e811 100644 --- a/library/cpp/messagebus/codegen.h +++ b/library/cpp/messagebus/codegen.h @@ -1,4 +1,4 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/config/codegen.h> - + diff --git a/library/cpp/messagebus/config/codegen.h b/library/cpp/messagebus/config/codegen.h index 5508fefc92..97ddada005 100644 --- a/library/cpp/messagebus/config/codegen.h +++ b/library/cpp/messagebus/config/codegen.h @@ -1,10 +1,10 @@ -#pragma once - -#define COMMA , - -#define STRUCT_FIELD_GEN(name, type, ...) type name; - -#define STRUCT_FIELD_INIT(name, type, defa) name(defa) -#define STRUCT_FIELD_INIT_DEFAULT(name, type, ...) name() - -#define STRUCT_FIELD_PRINT(name, ...) ss << #name << "=" << name << "\n"; +#pragma once + +#define COMMA , + +#define STRUCT_FIELD_GEN(name, type, ...) type name; + +#define STRUCT_FIELD_INIT(name, type, defa) name(defa) +#define STRUCT_FIELD_INIT_DEFAULT(name, type, ...) name() + +#define STRUCT_FIELD_PRINT(name, ...) ss << #name << "=" << name << "\n"; diff --git a/library/cpp/messagebus/config/defs.h b/library/cpp/messagebus/config/defs.h index 10820f83fd..92b1df9969 100644 --- a/library/cpp/messagebus/config/defs.h +++ b/library/cpp/messagebus/config/defs.h @@ -1,23 +1,23 @@ -#pragma once +#pragma once // unique tag to fix pragma once gcc glueing: ./library/cpp/messagebus/defs.h - + #include "codegen.h" #include "netaddr.h" #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> -#include <util/generic/list.h> +#include <util/generic/list.h> #include <utility> - -// For historical reasons TCrawlerModule need to access -// APIs that should be private. -class TCrawlerModule; - -struct TDebugReceiverHandler; - -namespace NBus { + +// For historical reasons TCrawlerModule need to access +// APIs that should be private. +class TCrawlerModule; + +struct TDebugReceiverHandler; + +namespace NBus { namespace NPrivate { class TAcceptor; struct TBusSessionImpl; @@ -27,15 +27,15 @@ namespace NBus { class TRemoteServerConnection; class TRemoteClientConnection; class TBusSyncSourceSessionImpl; - + struct TBusMessagePtrAndHeader; - + struct TSessionDumpStatus; - + struct TClientRequestImpl; - + } - + class TBusSession; struct TBusServerSession; struct TBusClientSession; @@ -47,36 +47,36 @@ namespace NBus { struct TBusQueueConfig; struct TBusSessionConfig; struct TBusHeader; - + class IThreadHandler; - + using TBusKey = ui64; using TBusMessageList = TList<TBusMessage*>; using TBusKeyVec = TVector<std::pair<TBusKey, TBusKey>>; - + using TBusMessageQueuePtr = TIntrusivePtr<TBusMessageQueue>; - + class TBusModule; - + using TBusData = TString; using TBusService = const char*; - + #define YBUS_KEYMIN TBusKey(0L) #define YBUS_KEYMAX TBusKey(-1L) #define YBUS_KEYLOCAL TBusKey(7L) -#define YBUS_KEYINVALID TBusKey(99999999L) - +#define YBUS_KEYINVALID TBusKey(99999999L) + // Check that generated id is valid for remote message inline bool IsBusKeyValid(TBusKey key) { return key != YBUS_KEYINVALID && key != YBUS_KEYMAX && key > YBUS_KEYLOCAL; } #define YBUS_VERSION 0 - + #define YBUS_INFINITE (1u << 30u) - + #define YBUS_STATUS_BASIC 0x0000 #define YBUS_STATUS_CONNS 0x0001 -#define YBUS_STATUS_INFLIGHT 0x0002 - +#define YBUS_STATUS_INFLIGHT 0x0002 + } diff --git a/library/cpp/messagebus/config/netaddr.cpp b/library/cpp/messagebus/config/netaddr.cpp index 3c6311ab15..962ac538e2 100644 --- a/library/cpp/messagebus/config/netaddr.cpp +++ b/library/cpp/messagebus/config/netaddr.cpp @@ -1,7 +1,7 @@ #include "netaddr.h" -#include <util/network/address.h> - +#include <util/network/address.h> + #include <cstdlib> namespace NBus { @@ -15,8 +15,8 @@ namespace NBus { return "EIP_VERSION_6"; } Y_FAIL(); - } - + } + int ToAddrFamily(EIpVersion ipVersion) { switch (ipVersion) { case EIP_VERSION_ANY: @@ -83,8 +83,8 @@ namespace NBus { default: Y_FAIL("unreachable"); } - } - + } + TAutoPtr<IRemoteAddr> MakeAddress(const TNetworkAddress& na, EIpVersion requireVersion, EIpVersion preferVersion) { TAutoPtr<IRemoteAddr> addr; for (TNetworkAddress::TIterator it = na.Begin(); it != na.End(); ++it) { @@ -95,9 +95,9 @@ namespace NBus { addr.Reset(new TNetworkAddressRef(na, &*it)); } } - } + } return addr; - } + } TAutoPtr<IRemoteAddr> MakeAddress(TStringBuf host, int port, EIpVersion requireVersion, EIpVersion preferVersion) { TString hostString(host); TNetworkAddress na(hostString, port); @@ -136,17 +136,17 @@ namespace NBus { if (!Ptr) { ythrow TNetAddr::TError() << "cannot resolve into " << Describe(requireVersion); } - } - + } + TNetAddr::TNetAddr(const TNetworkAddress& na, const TAddrInfo& ai) : Ptr(new TNetworkAddressRef(na, ai)) { } - + const sockaddr* TNetAddr::Addr() const { return Ptr->Addr(); } - + socklen_t TNetAddr::Len() const { return Ptr->Len(); } @@ -169,14 +169,14 @@ namespace NBus { bool TNetAddr::IsIpv6() const { return Ptr->Addr()->sa_family == AF_INET6; - } - + } + bool TNetAddr::operator==(const TNetAddr& rhs) const { return Ptr == rhs.Ptr || Compare(*Ptr, *rhs.Ptr); } - -} - + +} + template <> void Out<NBus::TNetAddr>(IOutputStream& out, const NBus::TNetAddr& addr) { Out<NAddr::IRemoteAddr>(out, addr); diff --git a/library/cpp/messagebus/config/netaddr.h b/library/cpp/messagebus/config/netaddr.h index 3f8c80cde5..b79c0cc355 100644 --- a/library/cpp/messagebus/config/netaddr.h +++ b/library/cpp/messagebus/config/netaddr.h @@ -29,7 +29,7 @@ namespace NBus { const char* ToCString(EIpVersion); int ToAddrFamily(EIpVersion); - + /// Hold referenced pointer to address description structure (ex. sockaddr_storage) /// It's make possible to work with IPv4 / IPv6 addresses simultaneously class TNetAddr: public IRemoteAddr { @@ -62,7 +62,7 @@ namespace NBus { private: TAtomicSharedPtr<IRemoteAddr> Ptr; }; - + using TSockAddrInVector = TVector<TNetAddr>; struct TNetAddrHostPortHash { diff --git a/library/cpp/messagebus/config/session_config.cpp b/library/cpp/messagebus/config/session_config.cpp index fefd61c8f4..fbbbb106c9 100644 --- a/library/cpp/messagebus/config/session_config.cpp +++ b/library/cpp/messagebus/config/session_config.cpp @@ -1,31 +1,31 @@ -#include "session_config.h" - +#include "session_config.h" + #include <util/generic/strbuf.h> #include <util/string/hex.h> - -using namespace NBus; - -TBusSessionConfig::TSecret::TSecret() - : TimeoutPeriod(TDuration::Seconds(1)) - , StatusFlushPeriod(TDuration::MilliSeconds(400)) -{ -} - -TBusSessionConfig::TBusSessionConfig() - : BUS_SESSION_CONFIG_MAP(STRUCT_FIELD_INIT, COMMA) + +using namespace NBus; + +TBusSessionConfig::TSecret::TSecret() + : TimeoutPeriod(TDuration::Seconds(1)) + , StatusFlushPeriod(TDuration::MilliSeconds(400)) +{ +} + +TBusSessionConfig::TBusSessionConfig() + : BUS_SESSION_CONFIG_MAP(STRUCT_FIELD_INIT, COMMA) { } - + TString TBusSessionConfig::PrintToString() const { - TStringStream ss; - BUS_SESSION_CONFIG_MAP(STRUCT_FIELD_PRINT, ) - return ss.Str(); -} - -static int ParseDurationForMessageBus(const char* option) { - return TDuration::Parse(option).MilliSeconds(); -} - + TStringStream ss; + BUS_SESSION_CONFIG_MAP(STRUCT_FIELD_PRINT, ) + return ss.Str(); +} + +static int ParseDurationForMessageBus(const char* option) { + return TDuration::Parse(option).MilliSeconds(); +} + static int ParseToSForMessageBus(const char* option) { int tos; TStringBuf str(option); @@ -42,21 +42,21 @@ static int ParseToSForMessageBus(const char* option) { template <class T> static T ParseWithKmgSuffixT(const char* option) { - TStringBuf str(option); + TStringBuf str(option); T multiplier = 1; if (str.EndsWith('k')) { - multiplier = 1024; + multiplier = 1024; str = str.Head(str.size() - 1); } else if (str.EndsWith('m')) { - multiplier = 1024 * 1024; + multiplier = 1024 * 1024; str = str.Head(str.size() - 1); } else if (str.EndsWith('g')) { - multiplier = 1024 * 1024 * 1024; + multiplier = 1024 * 1024 * 1024; str = str.Head(str.size() - 1); - } + } return FromString<T>(str) * multiplier; -} - +} + static ui64 ParseWithKmgSuffix(const char* option) { return ParseWithKmgSuffixT<ui64>(option); } @@ -90,7 +90,7 @@ void TBusSessionConfig::ConfigureLastGetopt(NLastGetopt::TOpts& opts, opts.AddLongOption(prefix + "max-in-flight") .RequiredArgument("COUNT") .DefaultValue(ToString(MaxInFlight)) - .StoreMappedResultT<const char*>(&MaxInFlight, &ParseWithKmgSuffix); + .StoreMappedResultT<const char*>(&MaxInFlight, &ParseWithKmgSuffix); opts.AddLongOption(prefix + "max-in-flight-by-size") .RequiredArgument("BYTES") .DefaultValue( @@ -116,7 +116,7 @@ void TBusSessionConfig::ConfigureLastGetopt(NLastGetopt::TOpts& opts, opts.AddLongOption(prefix + "max-buffer-size") .RequiredArgument("BYTES") .DefaultValue(ToString(MaxBufferSize)) - .StoreMappedResultT<const char*>(&MaxBufferSize, &ParseWithKmgSuffix); + .StoreMappedResultT<const char*>(&MaxBufferSize, &ParseWithKmgSuffix); opts.AddLongOption(prefix + "max-message-size") .RequiredArgument("BYTES") .DefaultValue(ToString(MaxMessageSize)) @@ -149,9 +149,9 @@ void TBusSessionConfig::ConfigureLastGetopt(NLastGetopt::TOpts& opts, opts.AddLongOption(prefix + "on-message-in-pool") .RequiredArgument("BOOL") .DefaultValue(ToString(ExecuteOnMessageInWorkerPool)) - .StoreResult(&ExecuteOnMessageInWorkerPool); + .StoreResult(&ExecuteOnMessageInWorkerPool); opts.AddLongOption(prefix + "on-reply-in-pool") .RequiredArgument("BOOL") .DefaultValue(ToString(ExecuteOnReplyInWorkerPool)) - .StoreResult(&ExecuteOnReplyInWorkerPool); -} + .StoreResult(&ExecuteOnReplyInWorkerPool); +} diff --git a/library/cpp/messagebus/config/session_config.h b/library/cpp/messagebus/config/session_config.h index 7b02dbcd4e..84753350a9 100644 --- a/library/cpp/messagebus/config/session_config.h +++ b/library/cpp/messagebus/config/session_config.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include "codegen.h" -#include "defs.h" - +#include "defs.h" + #include <library/cpp/getopt/last_getopt.h> - + #include <util/generic/string.h> - -namespace NBus { + +namespace NBus { #define BUS_SESSION_CONFIG_MAP(XX, comma) \ XX(Name, TString, "") \ comma \ @@ -35,31 +35,31 @@ namespace NBus { XX(ExecuteOnReplyInWorkerPool, bool, true) comma \ XX(ReusePort, bool, false) comma \ XX(ListenPort, unsigned, 0) /* TODO: server only */ - + //////////////////////////////////////////////////////////////////// /// \brief Configuration for client and server session struct TBusSessionConfig { BUS_SESSION_CONFIG_MAP(STRUCT_FIELD_GEN, ) - + struct TSecret { TDuration TimeoutPeriod; TDuration StatusFlushPeriod; - + TSecret(); }; - + // secret options are available, but you shouldn't probably use them TSecret Secret; - + /// initialized with default settings TBusSessionConfig(); - + TString PrintToString() const; - + void ConfigureLastGetopt(NLastGetopt::TOpts&, const TString& prefix = "mb-"); }; - + using TBusClientSessionConfig = TBusSessionConfig; using TBusServerSessionConfig = TBusSessionConfig; - -} // NBus + +} // NBus diff --git a/library/cpp/messagebus/connection.cpp b/library/cpp/messagebus/connection.cpp index 2b9a915150..07580ce18a 100644 --- a/library/cpp/messagebus/connection.cpp +++ b/library/cpp/messagebus/connection.cpp @@ -1,16 +1,16 @@ #include "connection.h" - -#include "remote_client_connection.h" - + +#include "remote_client_connection.h" + #include <util/generic/cast.h> - -using namespace NBus; -using namespace NBus::NPrivate; - -void TBusClientConnectionPtrOps::Ref(TBusClientConnection* c) { - return CheckedCast<TRemoteClientConnection*>(c)->Ref(); -} - -void TBusClientConnectionPtrOps::UnRef(TBusClientConnection* c) { - return CheckedCast<TRemoteClientConnection*>(c)->UnRef(); -} + +using namespace NBus; +using namespace NBus::NPrivate; + +void TBusClientConnectionPtrOps::Ref(TBusClientConnection* c) { + return CheckedCast<TRemoteClientConnection*>(c)->Ref(); +} + +void TBusClientConnectionPtrOps::UnRef(TBusClientConnection* c) { + return CheckedCast<TRemoteClientConnection*>(c)->UnRef(); +} diff --git a/library/cpp/messagebus/connection.h b/library/cpp/messagebus/connection.h index 9a2493162f..b1df64ddc1 100644 --- a/library/cpp/messagebus/connection.h +++ b/library/cpp/messagebus/connection.h @@ -1,22 +1,22 @@ -#pragma once - -#include "defs.h" -#include "message.h" - +#pragma once + +#include "defs.h" +#include "message.h" + #include <util/generic/ptr.h> -namespace NBus { +namespace NBus { struct TBusClientConnection { /// if you want to open connection early virtual void OpenConnection() = 0; - + /// Send message to the destination /// If addr is set then use it as destination. /// Takes ownership of addr (see ClearState method). virtual EMessageStatus SendMessage(TBusMessage* pMes, bool wait = false) = 0; - + virtual EMessageStatus SendMessageOneWay(TBusMessage* pMes, bool wait = false) = 0; - + /// Like SendMessage but cares about message template <typename T /* <: TBusMessage */> EMessageStatus SendMessageAutoPtr(const TAutoPtr<T>& mes, bool wait = false) { @@ -25,7 +25,7 @@ namespace NBus { Y_UNUSED(mes.Release()); return status; } - + /// Like SendMessageOneWay but cares about message template <typename T /* <: TBusMessage */> EMessageStatus SendMessageOneWayAutoPtr(const TAutoPtr<T>& mes, bool wait = false) { @@ -34,28 +34,28 @@ namespace NBus { Y_UNUSED(mes.Release()); return status; } - + EMessageStatus SendMessageMove(TBusMessageAutoPtr message, bool wait = false) { return SendMessageAutoPtr(message, wait); } - + EMessageStatus SendMessageOneWayMove(TBusMessageAutoPtr message, bool wait = false) { return SendMessageOneWayAutoPtr(message, wait); } - + // TODO: implement similar one-way methods - + virtual ~TBusClientConnection() { } - }; - + }; + namespace NPrivate { struct TBusClientConnectionPtrOps { static void Ref(TBusClientConnection*); static void UnRef(TBusClientConnection*); }; } - + using TBusClientConnectionPtr = TIntrusivePtr<TBusClientConnection, NPrivate::TBusClientConnectionPtrOps>; - -} + +} diff --git a/library/cpp/messagebus/coreconn.cpp b/library/cpp/messagebus/coreconn.cpp index be229a187a..d9411bb5db 100644 --- a/library/cpp/messagebus/coreconn.cpp +++ b/library/cpp/messagebus/coreconn.cpp @@ -1,6 +1,6 @@ #include "coreconn.h" -#include "remote_connection.h" +#include "remote_connection.h" #include <util/datetime/base.h> #include <util/generic/yexception.h> diff --git a/library/cpp/messagebus/coreconn.h b/library/cpp/messagebus/coreconn.h index a910f6f6d7..fca228d82e 100644 --- a/library/cpp/messagebus/coreconn.h +++ b/library/cpp/messagebus/coreconn.h @@ -24,9 +24,9 @@ #include <util/system/thread.h> #include <util/thread/lfqueue.h> -#include <deque> +#include <deque> #include <utility> - + #ifdef NO_ERROR #undef NO_ERROR #endif diff --git a/library/cpp/messagebus/coreconn_ut.cpp b/library/cpp/messagebus/coreconn_ut.cpp index 710a82b75a..beb6850f26 100644 --- a/library/cpp/messagebus/coreconn_ut.cpp +++ b/library/cpp/messagebus/coreconn_ut.cpp @@ -1,25 +1,25 @@ #include <library/cpp/testing/unittest/registar.h> -#include "coreconn.h" - -#include <util/generic/yexception.h> - +#include "coreconn.h" + +#include <util/generic/yexception.h> + Y_UNIT_TEST_SUITE(TMakeIpVersionTest) { - using namespace NBus; - + using namespace NBus; + Y_UNIT_TEST(IpV4Allowed) { - UNIT_ASSERT_EQUAL(MakeIpVersion(true, false), EIP_VERSION_4); - } - + UNIT_ASSERT_EQUAL(MakeIpVersion(true, false), EIP_VERSION_4); + } + Y_UNIT_TEST(IpV6Allowed) { - UNIT_ASSERT_EQUAL(MakeIpVersion(false, true), EIP_VERSION_6); - } - + UNIT_ASSERT_EQUAL(MakeIpVersion(false, true), EIP_VERSION_6); + } + Y_UNIT_TEST(AllAllowed) { - UNIT_ASSERT_EQUAL(MakeIpVersion(true, true), EIP_VERSION_ANY); - } - + UNIT_ASSERT_EQUAL(MakeIpVersion(true, true), EIP_VERSION_ANY); + } + Y_UNIT_TEST(NothingAllowed) { - UNIT_ASSERT_EXCEPTION(MakeIpVersion(false, false), yexception); - } -} + UNIT_ASSERT_EXCEPTION(MakeIpVersion(false, false), yexception); + } +} diff --git a/library/cpp/messagebus/debug_receiver/debug_receiver.cpp b/library/cpp/messagebus/debug_receiver/debug_receiver.cpp index dfd3d0f1c9..23b02d1003 100644 --- a/library/cpp/messagebus/debug_receiver/debug_receiver.cpp +++ b/library/cpp/messagebus/debug_receiver/debug_receiver.cpp @@ -1,42 +1,42 @@ #include "debug_receiver_handler.h" #include "debug_receiver_proto.h" - + #include <library/cpp/messagebus/ybus.h> #include <library/cpp/getopt/last_getopt.h> #include <library/cpp/lwtrace/all.h> - -using namespace NBus; - -int main(int argc, char** argv) { - NLWTrace::StartLwtraceFromEnv(); - - TBusQueueConfig queueConfig; - TBusServerSessionConfig sessionConfig; - - NLastGetopt::TOpts opts; - - queueConfig.ConfigureLastGetopt(opts); - sessionConfig.ConfigureLastGetopt(opts); - - opts.AddLongOption("port").Required().RequiredArgument("PORT").StoreResult(&sessionConfig.ListenPort); - - opts.SetFreeArgsMax(0); - - NLastGetopt::TOptsParseResult r(&opts, argc, argv); - - TBusMessageQueuePtr q(CreateMessageQueue(queueConfig)); - - TDebugReceiverProtocol proto; - TDebugReceiverHandler handler; - - TBusServerSessionPtr serverSession = TBusServerSession::Create(&proto, &handler, sessionConfig, q); - // TODO: race is here - handler.ServerSession = serverSession.Get(); - - for (;;) { - Sleep(TDuration::Hours(17)); - } - - return 0; -} + +using namespace NBus; + +int main(int argc, char** argv) { + NLWTrace::StartLwtraceFromEnv(); + + TBusQueueConfig queueConfig; + TBusServerSessionConfig sessionConfig; + + NLastGetopt::TOpts opts; + + queueConfig.ConfigureLastGetopt(opts); + sessionConfig.ConfigureLastGetopt(opts); + + opts.AddLongOption("port").Required().RequiredArgument("PORT").StoreResult(&sessionConfig.ListenPort); + + opts.SetFreeArgsMax(0); + + NLastGetopt::TOptsParseResult r(&opts, argc, argv); + + TBusMessageQueuePtr q(CreateMessageQueue(queueConfig)); + + TDebugReceiverProtocol proto; + TDebugReceiverHandler handler; + + TBusServerSessionPtr serverSession = TBusServerSession::Create(&proto, &handler, sessionConfig, q); + // TODO: race is here + handler.ServerSession = serverSession.Get(); + + for (;;) { + Sleep(TDuration::Hours(17)); + } + + return 0; +} diff --git a/library/cpp/messagebus/debug_receiver/debug_receiver_handler.cpp b/library/cpp/messagebus/debug_receiver/debug_receiver_handler.cpp index 7490ae17c7..05f99e94ca 100644 --- a/library/cpp/messagebus/debug_receiver/debug_receiver_handler.cpp +++ b/library/cpp/messagebus/debug_receiver/debug_receiver_handler.cpp @@ -1,20 +1,20 @@ #include "debug_receiver_handler.h" - -#include "debug_receiver_proto.h" - + +#include "debug_receiver_proto.h" + #include <util/generic/cast.h> #include <util/string/printf.h> - + void TDebugReceiverHandler::OnError(TAutoPtr<NBus::TBusMessage>, NBus::EMessageStatus status) { - Cerr << "error " << status << "\n"; -} - + Cerr << "error " << status << "\n"; +} + void TDebugReceiverHandler::OnMessage(NBus::TOnMessageContext& message) { - TDebugReceiverMessage* typedMessage = VerifyDynamicCast<TDebugReceiverMessage*>(message.GetMessage()); - Cerr << "type=" << typedMessage->GetHeader()->Type + TDebugReceiverMessage* typedMessage = VerifyDynamicCast<TDebugReceiverMessage*>(message.GetMessage()); + Cerr << "type=" << typedMessage->GetHeader()->Type << " size=" << typedMessage->GetHeader()->Size << " flags=" << Sprintf("0x%04x", (int)typedMessage->GetHeader()->FlagsInternal) << "\n"; - - message.ForgetRequest(); -} + + message.ForgetRequest(); +} diff --git a/library/cpp/messagebus/debug_receiver/debug_receiver_handler.h b/library/cpp/messagebus/debug_receiver/debug_receiver_handler.h index 5a280fb537..0aed6b9984 100644 --- a/library/cpp/messagebus/debug_receiver/debug_receiver_handler.h +++ b/library/cpp/messagebus/debug_receiver/debug_receiver_handler.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/ybus.h> - + struct TDebugReceiverHandler: public NBus::IBusServerHandler { - NBus::TBusServerSession* ServerSession; - + NBus::TBusServerSession* ServerSession; + void OnError(TAutoPtr<NBus::TBusMessage> pMessage, NBus::EMessageStatus status) override; void OnMessage(NBus::TOnMessageContext& message) override; -}; +}; diff --git a/library/cpp/messagebus/debug_receiver/debug_receiver_proto.cpp b/library/cpp/messagebus/debug_receiver/debug_receiver_proto.cpp index 50990dae03..0c74f9ecc3 100644 --- a/library/cpp/messagebus/debug_receiver/debug_receiver_proto.cpp +++ b/library/cpp/messagebus/debug_receiver/debug_receiver_proto.cpp @@ -1,20 +1,20 @@ -#include "debug_receiver_proto.h" - -using namespace NBus; - -TDebugReceiverProtocol::TDebugReceiverProtocol() - : TBusProtocol("debug receiver", 0) -{ -} - +#include "debug_receiver_proto.h" + +using namespace NBus; + +TDebugReceiverProtocol::TDebugReceiverProtocol() + : TBusProtocol("debug receiver", 0) +{ +} + void TDebugReceiverProtocol::Serialize(const NBus::TBusMessage*, TBuffer&) { Y_FAIL("it is receiver only"); -} - +} + TAutoPtr<NBus::TBusMessage> TDebugReceiverProtocol::Deserialize(ui16, TArrayRef<const char> payload) { - THolder<TDebugReceiverMessage> r(new TDebugReceiverMessage(ECreateUninitialized())); - - r->Payload.Append(payload.data(), payload.size()); - - return r.Release(); -} + THolder<TDebugReceiverMessage> r(new TDebugReceiverMessage(ECreateUninitialized())); + + r->Payload.Append(payload.data(), payload.size()); + + return r.Release(); +} diff --git a/library/cpp/messagebus/debug_receiver/debug_receiver_proto.h b/library/cpp/messagebus/debug_receiver/debug_receiver_proto.h index 90c01444fb..d34710dcf7 100644 --- a/library/cpp/messagebus/debug_receiver/debug_receiver_proto.h +++ b/library/cpp/messagebus/debug_receiver/debug_receiver_proto.h @@ -1,27 +1,27 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/ybus.h> - + struct TDebugReceiverMessage: public NBus::TBusMessage { - /// constructor to create messages on sending end + /// constructor to create messages on sending end TDebugReceiverMessage(ui16 type) : NBus::TBusMessage(type) { } - - /// constructor with serialzed data to examine the header + + /// constructor with serialzed data to examine the header TDebugReceiverMessage(NBus::ECreateUninitialized) : NBus::TBusMessage(NBus::ECreateUninitialized()) { } - - TBuffer Payload; -}; - + + TBuffer Payload; +}; + struct TDebugReceiverProtocol: public NBus::TBusProtocol { - TDebugReceiverProtocol(); - + TDebugReceiverProtocol(); + void Serialize(const NBus::TBusMessage* mess, TBuffer& data) override; - + TAutoPtr<NBus::TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) override; -}; +}; diff --git a/library/cpp/messagebus/debug_receiver/ya.make b/library/cpp/messagebus/debug_receiver/ya.make index dd498e41cb..f1b14d35bb 100644 --- a/library/cpp/messagebus/debug_receiver/ya.make +++ b/library/cpp/messagebus/debug_receiver/ya.make @@ -1,17 +1,17 @@ -PROGRAM(messagebus_debug_receiver) - +PROGRAM(messagebus_debug_receiver) + OWNER(g:messagebus) - -SRCS( - debug_receiver.cpp - debug_receiver_proto.cpp - debug_receiver_handler.cpp -) - -PEERDIR( + +SRCS( + debug_receiver.cpp + debug_receiver_proto.cpp + debug_receiver_handler.cpp +) + +PEERDIR( library/cpp/getopt library/cpp/lwtrace library/cpp/messagebus -) - -END() +) + +END() diff --git a/library/cpp/messagebus/defs.h b/library/cpp/messagebus/defs.h index 9d2e000dd6..cb553acc45 100644 --- a/library/cpp/messagebus/defs.h +++ b/library/cpp/messagebus/defs.h @@ -1,4 +1,4 @@ -#pragma once +#pragma once #include <library/cpp/messagebus/config/defs.h> - + diff --git a/library/cpp/messagebus/dummy_debugger.h b/library/cpp/messagebus/dummy_debugger.h index b614ee94a6..89a4e18716 100644 --- a/library/cpp/messagebus/dummy_debugger.h +++ b/library/cpp/messagebus/dummy_debugger.h @@ -1,9 +1,9 @@ -#pragma once - -#include <util/datetime/base.h> -#include <util/stream/output.h> - +#pragma once + +#include <util/datetime/base.h> +#include <util/stream/output.h> + #define MB_TRACE() \ do { \ - Cerr << TInstant::Now() << " " << __FILE__ << ":" << __LINE__ << " " << __FUNCTION__ << Endl; \ - } while (false) + Cerr << TInstant::Now() << " " << __FILE__ << ":" << __LINE__ << " " << __FUNCTION__ << Endl; \ + } while (false) diff --git a/library/cpp/messagebus/duration_histogram.cpp b/library/cpp/messagebus/duration_histogram.cpp index 5217314776..32a0001d41 100644 --- a/library/cpp/messagebus/duration_histogram.cpp +++ b/library/cpp/messagebus/duration_histogram.cpp @@ -1,74 +1,74 @@ #include "duration_histogram.h" #include <util/generic/singleton.h> -#include <util/stream/str.h> - -namespace { - ui64 SecondsRound(TDuration d) { - if (d.MilliSeconds() % 1000 >= 500) { - return d.Seconds() + 1; - } else { - return d.Seconds(); - } - } - - ui64 MilliSecondsRound(TDuration d) { - if (d.MicroSeconds() % 1000 >= 500) { - return d.MilliSeconds() + 1; - } else { - return d.MilliSeconds(); - } - } - - ui64 MinutesRound(TDuration d) { - if (d.Seconds() % 60 >= 30) { - return d.Minutes() + 1; - } else { - return d.Minutes(); - } - } - -} - -namespace { - struct TMarks { +#include <util/stream/str.h> + +namespace { + ui64 SecondsRound(TDuration d) { + if (d.MilliSeconds() % 1000 >= 500) { + return d.Seconds() + 1; + } else { + return d.Seconds(); + } + } + + ui64 MilliSecondsRound(TDuration d) { + if (d.MicroSeconds() % 1000 >= 500) { + return d.MilliSeconds() + 1; + } else { + return d.MilliSeconds(); + } + } + + ui64 MinutesRound(TDuration d) { + if (d.Seconds() % 60 >= 30) { + return d.Minutes() + 1; + } else { + return d.Minutes(); + } + } + +} + +namespace { + struct TMarks { std::array<TDuration, TDurationHistogram::Buckets> Marks; - - TMarks() { - Marks[0] = TDuration::Zero(); - for (unsigned i = 1; i < TDurationHistogram::Buckets; ++i) { - if (i >= TDurationHistogram::SecondBoundary) { - Marks[i] = TDuration::Seconds(1) * (1 << (i - TDurationHistogram::SecondBoundary)); - } else { - Marks[i] = TDuration::Seconds(1) / (1 << (TDurationHistogram::SecondBoundary - i)); - } - } - } - }; -} - + + TMarks() { + Marks[0] = TDuration::Zero(); + for (unsigned i = 1; i < TDurationHistogram::Buckets; ++i) { + if (i >= TDurationHistogram::SecondBoundary) { + Marks[i] = TDuration::Seconds(1) * (1 << (i - TDurationHistogram::SecondBoundary)); + } else { + Marks[i] = TDuration::Seconds(1) / (1 << (TDurationHistogram::SecondBoundary - i)); + } + } + } + }; +} + TString TDurationHistogram::LabelBefore(unsigned i) { Y_VERIFY(i < Buckets); - - TDuration d = Singleton<TMarks>()->Marks[i]; - - TStringStream ss; - if (d == TDuration::Zero()) { - ss << "0"; - } else if (d < TDuration::Seconds(1)) { - ss << MilliSecondsRound(d) << "ms"; - } else if (d < TDuration::Minutes(1)) { - ss << SecondsRound(d) << "s"; - } else { - ss << MinutesRound(d) << "m"; - } - return ss.Str(); -} - + + TDuration d = Singleton<TMarks>()->Marks[i]; + + TStringStream ss; + if (d == TDuration::Zero()) { + ss << "0"; + } else if (d < TDuration::Seconds(1)) { + ss << MilliSecondsRound(d) << "ms"; + } else if (d < TDuration::Minutes(1)) { + ss << SecondsRound(d) << "s"; + } else { + ss << MinutesRound(d) << "m"; + } + return ss.Str(); +} + TString TDurationHistogram::PrintToString() const { - TStringStream ss; + TStringStream ss; for (auto time : Times) { ss << time << "\n"; - } - return ss.Str(); -} + } + return ss.Str(); +} diff --git a/library/cpp/messagebus/duration_histogram.h b/library/cpp/messagebus/duration_histogram.h index bda8c85704..ed060b0101 100644 --- a/library/cpp/messagebus/duration_histogram.h +++ b/library/cpp/messagebus/duration_histogram.h @@ -1,45 +1,45 @@ -#pragma once - -#include <util/datetime/base.h> +#pragma once + +#include <util/datetime/base.h> #include <util/generic/bitops.h> #include <util/generic/string.h> - + #include <array> - -struct TDurationHistogram { - static const unsigned Buckets = 20; + +struct TDurationHistogram { + static const unsigned Buckets = 20; std::array<ui64, Buckets> Times; - - static const unsigned SecondBoundary = 11; - - TDurationHistogram() { + + static const unsigned SecondBoundary = 11; + + TDurationHistogram() { Times.fill(0); - } - - static unsigned BucketFor(TDuration d) { - ui64 units = d.MicroSeconds() * (1 << SecondBoundary) / 1000000; - if (units == 0) { - return 0; - } - unsigned bucket = GetValueBitCount(units) - 1; - if (bucket >= Buckets) { - bucket = Buckets - 1; - } - return bucket; - } - - void AddTime(TDuration d) { - Times[BucketFor(d)] += 1; - } - - TDurationHistogram& operator+=(const TDurationHistogram& that) { - for (unsigned i = 0; i < Times.size(); ++i) { - Times[i] += that.Times[i]; - } - return *this; - } - + } + + static unsigned BucketFor(TDuration d) { + ui64 units = d.MicroSeconds() * (1 << SecondBoundary) / 1000000; + if (units == 0) { + return 0; + } + unsigned bucket = GetValueBitCount(units) - 1; + if (bucket >= Buckets) { + bucket = Buckets - 1; + } + return bucket; + } + + void AddTime(TDuration d) { + Times[BucketFor(d)] += 1; + } + + TDurationHistogram& operator+=(const TDurationHistogram& that) { + for (unsigned i = 0; i < Times.size(); ++i) { + Times[i] += that.Times[i]; + } + return *this; + } + static TString LabelBefore(unsigned i); - + TString PrintToString() const; -}; +}; diff --git a/library/cpp/messagebus/duration_histogram_ut.cpp b/library/cpp/messagebus/duration_histogram_ut.cpp index 5e2a194b6d..01bcc095e9 100644 --- a/library/cpp/messagebus/duration_histogram_ut.cpp +++ b/library/cpp/messagebus/duration_histogram_ut.cpp @@ -1,38 +1,38 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "duration_histogram.h" - + +#include "duration_histogram.h" + Y_UNIT_TEST_SUITE(TDurationHistogramTest) { Y_UNIT_TEST(BucketFor) { UNIT_ASSERT_VALUES_EQUAL(0u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(0))); UNIT_ASSERT_VALUES_EQUAL(0u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(1))); UNIT_ASSERT_VALUES_EQUAL(0u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(900))); - UNIT_ASSERT_VALUES_EQUAL(1u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(1500))); - UNIT_ASSERT_VALUES_EQUAL(2u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(2500))); - - unsigned sb = TDurationHistogram::SecondBoundary; - + UNIT_ASSERT_VALUES_EQUAL(1u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(1500))); + UNIT_ASSERT_VALUES_EQUAL(2u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(2500))); + + unsigned sb = TDurationHistogram::SecondBoundary; + UNIT_ASSERT_VALUES_EQUAL(sb - 1, TDurationHistogram::BucketFor(TDuration::MilliSeconds(999))); - UNIT_ASSERT_VALUES_EQUAL(sb, TDurationHistogram::BucketFor(TDuration::MilliSeconds(1000))); - UNIT_ASSERT_VALUES_EQUAL(sb, TDurationHistogram::BucketFor(TDuration::MilliSeconds(1001))); - - UNIT_ASSERT_VALUES_EQUAL(TDurationHistogram::Buckets - 1, TDurationHistogram::BucketFor(TDuration::Hours(1))); - } - + UNIT_ASSERT_VALUES_EQUAL(sb, TDurationHistogram::BucketFor(TDuration::MilliSeconds(1000))); + UNIT_ASSERT_VALUES_EQUAL(sb, TDurationHistogram::BucketFor(TDuration::MilliSeconds(1001))); + + UNIT_ASSERT_VALUES_EQUAL(TDurationHistogram::Buckets - 1, TDurationHistogram::BucketFor(TDuration::Hours(1))); + } + Y_UNIT_TEST(Simple) { - TDurationHistogram h1; - h1.AddTime(TDuration::MicroSeconds(1)); - UNIT_ASSERT_VALUES_EQUAL(1u, h1.Times.front()); - - TDurationHistogram h2; - h1.AddTime(TDuration::Hours(1)); - UNIT_ASSERT_VALUES_EQUAL(1u, h1.Times.back()); - } - + TDurationHistogram h1; + h1.AddTime(TDuration::MicroSeconds(1)); + UNIT_ASSERT_VALUES_EQUAL(1u, h1.Times.front()); + + TDurationHistogram h2; + h1.AddTime(TDuration::Hours(1)); + UNIT_ASSERT_VALUES_EQUAL(1u, h1.Times.back()); + } + Y_UNIT_TEST(LabelFor) { - for (unsigned i = 0; i < TDurationHistogram::Buckets; ++i) { - TDurationHistogram::LabelBefore(i); - //Cerr << TDurationHistogram::LabelBefore(i) << "\n"; - } - } -} + for (unsigned i = 0; i < TDurationHistogram::Buckets; ++i) { + TDurationHistogram::LabelBefore(i); + //Cerr << TDurationHistogram::LabelBefore(i) << "\n"; + } + } +} diff --git a/library/cpp/messagebus/event_loop.cpp b/library/cpp/messagebus/event_loop.cpp index b82bd023e5..f685135bed 100644 --- a/library/cpp/messagebus/event_loop.cpp +++ b/library/cpp/messagebus/event_loop.cpp @@ -1,8 +1,8 @@ #include "event_loop.h" - + #include "network.h" -#include "thread_extra.h" - +#include "thread_extra.h" + #include <util/generic/hash.h> #include <util/network/pair.h> #include <util/network/poller.h> @@ -10,19 +10,19 @@ #include <util/system/mutex.h> #include <util/system/thread.h> #include <util/system/yassert.h> -#include <util/thread/lfqueue.h> +#include <util/thread/lfqueue.h> #include <errno.h> - + using namespace NEventLoop; namespace { - enum ERunningState { - EVENT_LOOP_CREATED, - EVENT_LOOP_RUNNING, - EVENT_LOOP_STOPPED, - }; - + enum ERunningState { + EVENT_LOOP_CREATED, + EVENT_LOOP_RUNNING, + EVENT_LOOP_STOPPED, + }; + enum EOperation { OP_READ = 1, OP_WRITE = 2, @@ -32,8 +32,8 @@ namespace { class TChannel::TImpl { public: - TImpl(TEventLoop::TImpl* eventLoop, TSocket socket, TEventHandlerPtr, void* cookie); - ~TImpl(); + TImpl(TEventLoop::TImpl* eventLoop, TSocket socket, TEventHandlerPtr, void* cookie); + ~TImpl(); void EnableRead(); void DisableRead(); @@ -43,48 +43,48 @@ public: void Unregister(); SOCKET GetSocket() const; - TSocket GetSocketPtr() const; + TSocket GetSocketPtr() const; + + void Update(int pollerFlags, bool enable); + void CallHandler(); - void Update(int pollerFlags, bool enable); - void CallHandler(); - TEventLoop::TImpl* EventLoop; - TSocket Socket; - TEventHandlerPtr EventHandler; - void* Cookie; - - TMutex Mutex; - - int CurrentFlags; - bool Close; + TSocket Socket; + TEventHandlerPtr EventHandler; + void* Cookie; + + TMutex Mutex; + + int CurrentFlags; + bool Close; }; class TEventLoop::TImpl { public: - TImpl(const char* name); + TImpl(const char* name); void Run(); void Wakeup(); void Stop(); - TChannelPtr Register(TSocket socket, TEventHandlerPtr eventHandler, void* cookie); + TChannelPtr Register(TSocket socket, TEventHandlerPtr eventHandler, void* cookie); void Unregister(SOCKET socket); typedef THashMap<SOCKET, TChannelPtr> TData; - void AddToPoller(SOCKET socket, void* cookie, int flags); + void AddToPoller(SOCKET socket, void* cookie, int flags); - TMutex Mutex; + TMutex Mutex; + + const char* Name; - const char* Name; - TAtomic RunningState; TAtomic StopSignal; TSystemEvent StoppedEvent; TData Data; - TLockFreeQueue<SOCKET> SocketsToRemove; - + TLockFreeQueue<SOCKET> SocketsToRemove; + TSocketPoller Poller; TSocketHolder WakeupReadSocket; TSocketHolder WakeupWriteSocket; @@ -117,17 +117,17 @@ SOCKET TChannel::GetSocket() const { return Impl->GetSocket(); } -TSocket TChannel::GetSocketPtr() const { - return Impl->GetSocketPtr(); -} - -TChannel::TChannel(TImpl* impl) - : Impl(impl) +TSocket TChannel::GetSocketPtr() const { + return Impl->GetSocketPtr(); +} + +TChannel::TChannel(TImpl* impl) + : Impl(impl) { } -TEventLoop::TEventLoop(const char* name) - : Impl(new TImpl(name)) +TEventLoop::TEventLoop(const char* name) + : Impl(new TImpl(name)) { } @@ -142,126 +142,126 @@ void TEventLoop::Stop() { Impl->Stop(); } -bool TEventLoop::IsRunning() { +bool TEventLoop::IsRunning() { return AtomicGet(Impl->RunningState) == EVENT_LOOP_RUNNING; -} - -TChannelPtr TEventLoop::Register(TSocket socket, TEventHandlerPtr eventHandler, void* cookie) { - return Impl->Register(socket, eventHandler, cookie); } -TChannel::TImpl::TImpl(TEventLoop::TImpl* eventLoop, TSocket socket, TEventHandlerPtr eventHandler, void* cookie) +TChannelPtr TEventLoop::Register(TSocket socket, TEventHandlerPtr eventHandler, void* cookie) { + return Impl->Register(socket, eventHandler, cookie); +} + +TChannel::TImpl::TImpl(TEventLoop::TImpl* eventLoop, TSocket socket, TEventHandlerPtr eventHandler, void* cookie) : EventLoop(eventLoop) , Socket(socket) - , EventHandler(eventHandler) - , Cookie(cookie) - , CurrentFlags(0) - , Close(false) + , EventHandler(eventHandler) + , Cookie(cookie) + , CurrentFlags(0) + , Close(false) { } -TChannel::TImpl::~TImpl() { +TChannel::TImpl::~TImpl() { Y_ASSERT(Close); -} - +} + void TChannel::TImpl::EnableRead() { - Update(OP_READ, true); + Update(OP_READ, true); } void TChannel::TImpl::DisableRead() { - Update(OP_READ, false); + Update(OP_READ, false); } void TChannel::TImpl::EnableWrite() { - Update(OP_WRITE, true); + Update(OP_WRITE, true); } void TChannel::TImpl::DisableWrite() { - Update(OP_WRITE, false); + Update(OP_WRITE, false); } void TChannel::TImpl::Unregister() { - TGuard<TMutex> guard(Mutex); - - if (Close) { - return; - } - - Close = true; - if (CurrentFlags != 0) { - EventLoop->Poller.Unwait(Socket); - CurrentFlags = 0; - } - EventHandler.Drop(); - - EventLoop->SocketsToRemove.Enqueue(Socket); - EventLoop->Wakeup(); -} - -void TChannel::TImpl::Update(int flags, bool enable) { - TGuard<TMutex> guard(Mutex); - - if (Close) { - return; - } - - int newFlags = enable ? (CurrentFlags | flags) : (CurrentFlags & ~flags); - - if (CurrentFlags == newFlags) { - return; - } - - if (!newFlags) { - EventLoop->Poller.Unwait(Socket); - } else { - void* cookie = reinterpret_cast<void*>(this); - EventLoop->AddToPoller(Socket, cookie, newFlags); - } - - CurrentFlags = newFlags; -} - + TGuard<TMutex> guard(Mutex); + + if (Close) { + return; + } + + Close = true; + if (CurrentFlags != 0) { + EventLoop->Poller.Unwait(Socket); + CurrentFlags = 0; + } + EventHandler.Drop(); + + EventLoop->SocketsToRemove.Enqueue(Socket); + EventLoop->Wakeup(); +} + +void TChannel::TImpl::Update(int flags, bool enable) { + TGuard<TMutex> guard(Mutex); + + if (Close) { + return; + } + + int newFlags = enable ? (CurrentFlags | flags) : (CurrentFlags & ~flags); + + if (CurrentFlags == newFlags) { + return; + } + + if (!newFlags) { + EventLoop->Poller.Unwait(Socket); + } else { + void* cookie = reinterpret_cast<void*>(this); + EventLoop->AddToPoller(Socket, cookie, newFlags); + } + + CurrentFlags = newFlags; +} + SOCKET TChannel::TImpl::GetSocket() const { return Socket; } -TSocket TChannel::TImpl::GetSocketPtr() const { - return Socket; -} - -void TChannel::TImpl::CallHandler() { - TEventHandlerPtr handler; - - { - TGuard<TMutex> guard(Mutex); - - // other thread may have re-added socket to epoll - // so even if CurrentFlags is 0, epoll may fire again - // so please use non-blocking operations - CurrentFlags = 0; - - if (Close) { - return; - } - - handler = EventHandler; - } - - if (!!handler) { - handler->HandleEvent(Socket, Cookie); - } -} - -TEventLoop::TImpl::TImpl(const char* name) - : Name(name) - , RunningState(EVENT_LOOP_CREATED) +TSocket TChannel::TImpl::GetSocketPtr() const { + return Socket; +} + +void TChannel::TImpl::CallHandler() { + TEventHandlerPtr handler; + + { + TGuard<TMutex> guard(Mutex); + + // other thread may have re-added socket to epoll + // so even if CurrentFlags is 0, epoll may fire again + // so please use non-blocking operations + CurrentFlags = 0; + + if (Close) { + return; + } + + handler = EventHandler; + } + + if (!!handler) { + handler->HandleEvent(Socket, Cookie); + } +} + +TEventLoop::TImpl::TImpl(const char* name) + : Name(name) + , RunningState(EVENT_LOOP_CREATED) , StopSignal(0) { SOCKET wakeupSockets[2]; - if (SocketPair(wakeupSockets) < 0) { + if (SocketPair(wakeupSockets) < 0) { Y_FAIL("failed to create socket pair for wakeup sockets: %s", LastSystemErrorText()); - } + } TSocketHolder wakeupReadSocket(wakeupSockets[0]); TSocketHolder wakeupWriteSocket(wakeupSockets[1]); @@ -279,91 +279,91 @@ TEventLoop::TImpl::TImpl(const char* name) void TEventLoop::TImpl::Run() { bool res = AtomicCas(&RunningState, EVENT_LOOP_RUNNING, EVENT_LOOP_CREATED); Y_VERIFY(res, "Invalid mbus event loop state"); - - if (!!Name) { + + if (!!Name) { SetCurrentThreadName(Name); - } - + } + while (AtomicGet(StopSignal) == 0) { void* cookies[1024]; const size_t count = Poller.WaitI(cookies, Y_ARRAY_SIZE(cookies)); void** end = cookies + count; for (void** c = cookies; c != end; ++c) { - TChannel::TImpl* s = reinterpret_cast<TChannel::TImpl*>(*c); + TChannel::TImpl* s = reinterpret_cast<TChannel::TImpl*>(*c); - if (*c == this) { + if (*c == this) { char buf[0x1000]; - if (NBus::NPrivate::SocketRecv(WakeupReadSocket, buf) < 0) { + if (NBus::NPrivate::SocketRecv(WakeupReadSocket, buf) < 0) { Y_FAIL("failed to recv from wakeup socket: %s", LastSystemErrorText()); - } + } continue; } - s->CallHandler(); + s->CallHandler(); } - + SOCKET socket = -1; while (SocketsToRemove.Dequeue(&socket)) { - TGuard<TMutex> guard(Mutex); + TGuard<TMutex> guard(Mutex); Y_VERIFY(Data.erase(socket) == 1, "must be removed once"); - } + } } - - { - TGuard<TMutex> guard(Mutex); + + { + TGuard<TMutex> guard(Mutex); for (auto& it : Data) { it.second->Unregister(); - } - - // release file descriptors - Data.clear(); - } - + } + + // release file descriptors + Data.clear(); + } + res = AtomicCas(&RunningState, EVENT_LOOP_STOPPED, EVENT_LOOP_RUNNING); - + Y_VERIFY(res); - StoppedEvent.Signal(); + StoppedEvent.Signal(); } void TEventLoop::TImpl::Stop() { AtomicSet(StopSignal, 1); - + if (AtomicGet(RunningState) == EVENT_LOOP_RUNNING) { Wakeup(); - + StoppedEvent.WaitI(); } } -TChannelPtr TEventLoop::TImpl::Register(TSocket socket, TEventHandlerPtr eventHandler, void* cookie) { +TChannelPtr TEventLoop::TImpl::Register(TSocket socket, TEventHandlerPtr eventHandler, void* cookie) { Y_VERIFY(socket != INVALID_SOCKET, "must be a valid socket"); - - TChannelPtr channel = new TChannel(new TChannel::TImpl(this, socket, eventHandler, cookie)); - TGuard<TMutex> guard(Mutex); + TChannelPtr channel = new TChannel(new TChannel::TImpl(this, socket, eventHandler, cookie)); + + TGuard<TMutex> guard(Mutex); Y_VERIFY(Data.insert(std::make_pair(socket, channel)).second, "must not be already inserted"); - return channel; + return channel; } void TEventLoop::TImpl::Wakeup() { if (NBus::NPrivate::SocketSend(WakeupWriteSocket, TArrayRef<const char>("", 1)) < 0) { - if (LastSystemError() != EAGAIN) { + if (LastSystemError() != EAGAIN) { Y_FAIL("failed to send to wakeup socket: %s", LastSystemErrorText()); - } + } } } void TEventLoop::TImpl::AddToPoller(SOCKET socket, void* cookie, int flags) { if (flags == OP_READ) { - Poller.WaitReadOneShot(socket, cookie); + Poller.WaitReadOneShot(socket, cookie); } else if (flags == OP_WRITE) { - Poller.WaitWriteOneShot(socket, cookie); + Poller.WaitWriteOneShot(socket, cookie); } else if (flags == OP_READ_WRITE) { - Poller.WaitReadWriteOneShot(socket, cookie); + Poller.WaitReadWriteOneShot(socket, cookie); } else { Y_FAIL("Wrong flags: %d", int(flags)); } diff --git a/library/cpp/messagebus/event_loop.h b/library/cpp/messagebus/event_loop.h index b313ba02a4..d5b0a53b0c 100644 --- a/library/cpp/messagebus/event_loop.h +++ b/library/cpp/messagebus/event_loop.h @@ -3,12 +3,12 @@ #include <util/generic/object_counter.h> #include <util/generic/ptr.h> #include <util/network/init.h> -#include <util/network/socket.h> +#include <util/network/socket.h> namespace NEventLoop { struct IEventHandler : public TAtomicRefCount<IEventHandler> { - virtual void HandleEvent(SOCKET socket, void* cookie) = 0; + virtual void HandleEvent(SOCKET socket, void* cookie) = 0; virtual ~IEventHandler() { } }; @@ -17,8 +17,8 @@ namespace NEventLoop { class TEventLoop; - // TODO: make TChannel itself a pointer - // to avoid confusion with Drop and Unregister + // TODO: make TChannel itself a pointer + // to avoid confusion with Drop and Unregister class TChannel : public TAtomicRefCount<TChannel> { public: @@ -32,15 +32,15 @@ namespace NEventLoop { void Unregister(); SOCKET GetSocket() const; - TSocket GetSocketPtr() const; + TSocket GetSocketPtr() const; private: class TImpl; friend class TEventLoop; - TObjectCounter<TChannel> ObjectCounter; - - TChannel(TImpl*); + TObjectCounter<TChannel> ObjectCounter; + + TChannel(TImpl*); private: THolder<TImpl> Impl; @@ -55,7 +55,7 @@ namespace NEventLoop { void Run(); void Stop(); - bool IsRunning(); + bool IsRunning(); TChannelPtr Register(TSocket socket, TEventHandlerPtr, void* cookie = nullptr); @@ -63,8 +63,8 @@ namespace NEventLoop { class TImpl; friend class TChannel; - TObjectCounter<TEventLoop> ObjectCounter; - + TObjectCounter<TEventLoop> ObjectCounter; + private: THolder<TImpl> Impl; }; diff --git a/library/cpp/messagebus/extra_ref.h b/library/cpp/messagebus/extra_ref.h index d20b9b6aa1..2927123266 100644 --- a/library/cpp/messagebus/extra_ref.h +++ b/library/cpp/messagebus/extra_ref.h @@ -1,36 +1,36 @@ -#pragma once - -#include <util/system/yassert.h> - -class TExtraRef { - TAtomic Holds; - -public: +#pragma once + +#include <util/system/yassert.h> + +class TExtraRef { + TAtomic Holds; + +public: TExtraRef() : Holds(false) { } - ~TExtraRef() { + ~TExtraRef() { Y_VERIFY(!Holds); - } - - template <typename TThis> - void Retain(TThis* thiz) { - if (AtomicGet(Holds)) { - return; - } - if (AtomicCas(&Holds, 1, 0)) { - thiz->Ref(); - } - } - - template <typename TThis> - void Release(TThis* thiz) { - if (!AtomicGet(Holds)) { - return; - } - if (AtomicCas(&Holds, 0, 1)) { - thiz->UnRef(); - } - } -}; + } + + template <typename TThis> + void Retain(TThis* thiz) { + if (AtomicGet(Holds)) { + return; + } + if (AtomicCas(&Holds, 1, 0)) { + thiz->Ref(); + } + } + + template <typename TThis> + void Release(TThis* thiz) { + if (!AtomicGet(Holds)) { + return; + } + if (AtomicCas(&Holds, 0, 1)) { + thiz->UnRef(); + } + } +}; diff --git a/library/cpp/messagebus/futex_like.cpp b/library/cpp/messagebus/futex_like.cpp index c0d5b4f4ec..7f965126db 100644 --- a/library/cpp/messagebus/futex_like.cpp +++ b/library/cpp/messagebus/futex_like.cpp @@ -1,55 +1,55 @@ -#include <util/system/platform.h> - -#ifdef _linux_ -#include <sys/syscall.h> -#include <linux/futex.h> +#include <util/system/platform.h> + +#ifdef _linux_ +#include <sys/syscall.h> +#include <linux/futex.h> #if !defined(SYS_futex) #define SYS_futex __NR_futex -#endif #endif - -#include <errno.h> - -#include <util/system/yassert.h> - -#include "futex_like.h" - -#ifdef _linux_ -namespace { +#endif + +#include <errno.h> + +#include <util/system/yassert.h> + +#include "futex_like.h" + +#ifdef _linux_ +namespace { int futex(int* uaddr, int op, int val, const struct timespec* timeout, int* uaddr2, int val3) { - return syscall(SYS_futex, uaddr, op, val, timeout, uaddr2, val3); - } -} -#endif - -void TFutexLike::Wake(size_t count) { + return syscall(SYS_futex, uaddr, op, val, timeout, uaddr2, val3); + } +} +#endif + +void TFutexLike::Wake(size_t count) { Y_ASSERT(count > 0); -#ifdef _linux_ - if (count > unsigned(Max<int>())) { - count = Max<int>(); - } +#ifdef _linux_ + if (count > unsigned(Max<int>())) { + count = Max<int>(); + } int r = futex(&Value, FUTEX_WAKE, count, nullptr, nullptr, 0); Y_VERIFY(r >= 0, "futex_wake failed: %s", strerror(errno)); -#else - TGuard<TMutex> guard(Mutex); - if (count == 1) { - CondVar.Signal(); - } else { - CondVar.BroadCast(); - } -#endif -} - -void TFutexLike::Wait(int expected) { -#ifdef _linux_ +#else + TGuard<TMutex> guard(Mutex); + if (count == 1) { + CondVar.Signal(); + } else { + CondVar.BroadCast(); + } +#endif +} + +void TFutexLike::Wait(int expected) { +#ifdef _linux_ int r = futex(&Value, FUTEX_WAIT, expected, nullptr, nullptr, 0); Y_VERIFY(r >= 0 || errno == EWOULDBLOCK, "futex_wait failed: %s", strerror(errno)); -#else - TGuard<TMutex> guard(Mutex); - if (expected == Get()) { - CondVar.WaitI(Mutex); - } -#endif -} +#else + TGuard<TMutex> guard(Mutex); + if (expected == Get()) { + CondVar.WaitI(Mutex); + } +#endif +} diff --git a/library/cpp/messagebus/futex_like.h b/library/cpp/messagebus/futex_like.h index d4ac486caf..31d60c60f1 100644 --- a/library/cpp/messagebus/futex_like.h +++ b/library/cpp/messagebus/futex_like.h @@ -1,86 +1,86 @@ -#pragma once - +#pragma once + #include <util/system/condvar.h> #include <util/system/mutex.h> -#include <util/system/platform.h> - -class TFutexLike { -private: -#ifdef _linux_ - int Value; -#else - TAtomic Value; - TMutex Mutex; - TCondVar CondVar; -#endif - -public: - TFutexLike() - : Value(0) +#include <util/system/platform.h> + +class TFutexLike { +private: +#ifdef _linux_ + int Value; +#else + TAtomic Value; + TMutex Mutex; + TCondVar CondVar; +#endif + +public: + TFutexLike() + : Value(0) { } - - int AddAndGet(int add) { -#ifdef _linux_ - //return __atomic_add_fetch(&Value, add, __ATOMIC_SEQ_CST); - return __sync_add_and_fetch(&Value, add); -#else - return AtomicAdd(Value, add); -#endif - } - - int GetAndAdd(int add) { - return AddAndGet(add) - add; - } - -// until we have modern GCC -#if 0 - int GetAndSet(int newValue) { -#ifdef _linux_ - return __atomic_exchange_n(&Value, newValue, __ATOMIC_SEQ_CST); -#else + + int AddAndGet(int add) { +#ifdef _linux_ + //return __atomic_add_fetch(&Value, add, __ATOMIC_SEQ_CST); + return __sync_add_and_fetch(&Value, add); +#else + return AtomicAdd(Value, add); +#endif + } + + int GetAndAdd(int add) { + return AddAndGet(add) - add; + } + +// until we have modern GCC +#if 0 + int GetAndSet(int newValue) { +#ifdef _linux_ + return __atomic_exchange_n(&Value, newValue, __ATOMIC_SEQ_CST); +#else return AtomicSwap(&Value, newValue); -#endif - } -#endif - - int Get() { -#ifdef _linux_ - //return __atomic_load_n(&Value, __ATOMIC_SEQ_CST); - __sync_synchronize(); - return Value; -#else - return AtomicGet(Value); -#endif - } - - void Set(int newValue) { -#ifdef _linux_ - //__atomic_store_n(&Value, newValue, __ATOMIC_SEQ_CST); - Value = newValue; - __sync_synchronize(); -#else - AtomicSet(Value, newValue); -#endif - } - - int GetAndIncrement() { - return AddAndGet(1) - 1; - } - - int IncrementAndGet() { - return AddAndGet(1); - } - - int GetAndDecrement() { - return AddAndGet(-1) + 1; - } - - int DecrementAndGet() { - return AddAndGet(-1); - } - - void Wake(size_t count = Max<size_t>()); - - void Wait(int expected); -}; +#endif + } +#endif + + int Get() { +#ifdef _linux_ + //return __atomic_load_n(&Value, __ATOMIC_SEQ_CST); + __sync_synchronize(); + return Value; +#else + return AtomicGet(Value); +#endif + } + + void Set(int newValue) { +#ifdef _linux_ + //__atomic_store_n(&Value, newValue, __ATOMIC_SEQ_CST); + Value = newValue; + __sync_synchronize(); +#else + AtomicSet(Value, newValue); +#endif + } + + int GetAndIncrement() { + return AddAndGet(1) - 1; + } + + int IncrementAndGet() { + return AddAndGet(1); + } + + int GetAndDecrement() { + return AddAndGet(-1) + 1; + } + + int DecrementAndGet() { + return AddAndGet(-1); + } + + void Wake(size_t count = Max<size_t>()); + + void Wait(int expected); +}; diff --git a/library/cpp/messagebus/handler.cpp b/library/cpp/messagebus/handler.cpp index ec12bf0b8f..333bd52934 100644 --- a/library/cpp/messagebus/handler.cpp +++ b/library/cpp/messagebus/handler.cpp @@ -1,11 +1,11 @@ -#include "handler.h" +#include "handler.h" #include "remote_server_connection.h" #include "ybus.h" - -using namespace NBus; -using namespace NBus::NPrivate; - + +using namespace NBus; +using namespace NBus::NPrivate; + void IBusErrorHandler::OnError(TAutoPtr<TBusMessage> pMessage, EMessageStatus status) { Y_UNUSED(pMessage); Y_UNUSED(status); @@ -19,17 +19,17 @@ void IBusClientHandler::OnMessageSent(TBusMessage* pMessage) { void IBusClientHandler::OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage) { Y_UNUSED(pMessage); } - + void IBusClientHandler::OnClientConnectionEvent(const TClientConnectionEvent&) { } - + void TOnMessageContext::ForgetRequest() { - Session->ForgetRequest(Ident); -} - + Session->ForgetRequest(Ident); +} + TNetAddr TOnMessageContext::GetPeerAddrNetAddr() const { - return Ident.GetNetAddr(); -} + return Ident.GetNetAddr(); +} bool TOnMessageContext::IsConnectionAlive() const { return !!Ident.Connection && Ident.Connection->IsAlive(); diff --git a/library/cpp/messagebus/handler.h b/library/cpp/messagebus/handler.h index d9df33814b..60002c68a6 100644 --- a/library/cpp/messagebus/handler.h +++ b/library/cpp/messagebus/handler.h @@ -1,44 +1,44 @@ -#pragma once - -#include "defs.h" +#pragma once + +#include "defs.h" #include "message.h" -#include "message_status.h" -#include "use_after_free_checker.h" -#include "use_count_checker.h" - +#include "message_status.h" +#include "use_after_free_checker.h" +#include "use_count_checker.h" + #include <util/generic/noncopyable.h> -namespace NBus { +namespace NBus { ///////////////////////////////////////////////////////////////// /// \brief Interface to message bus handler - + struct IBusErrorHandler { friend struct ::NBus::NPrivate::TBusSessionImpl; - + private: TUseAfterFreeChecker UseAfterFreeChecker; TUseCountChecker UseCountChecker; - + public: /// called when message or reply can't be delivered virtual void OnError(TAutoPtr<TBusMessage> pMessage, EMessageStatus status); - + virtual ~IBusErrorHandler() { } }; - + class TClientConnectionEvent { public: enum EType { CONNECTED, DISCONNECTED, }; - + private: EType Type; ui64 Id; TNetAddr Addr; - + public: TClientConnectionEvent(EType type, ui64 id, TNetAddr addr) : Type(type) @@ -56,15 +56,15 @@ namespace NBus { TNetAddr GetAddr() const { return Addr; } - }; - + }; + class TOnMessageContext : TNonCopyable { private: THolder<TBusMessage> Message; TBusIdentity Ident; // TODO: we don't need to store session, we have connection in ident TBusServerSession* Session; - + public: TOnMessageContext() : Session() @@ -76,53 +76,53 @@ namespace NBus { { Ident.Swap(ident); } - + bool IsInWork() const { return Ident.IsInWork(); } - + bool operator!() const { return !IsInWork(); } - + TBusMessage* GetMessage() { return Message.Get(); } - + TBusMessage* ReleaseMessage() { return Message.Release(); } - + TBusServerSession* GetSession() { return Session; } - + template <typename U /* <: TBusMessage */> EMessageStatus SendReplyAutoPtr(TAutoPtr<U>& rep); - + EMessageStatus SendReplyMove(TBusMessageAutoPtr response); - + void AckMessage(TBusIdentity& ident); - + void ForgetRequest(); - + void Swap(TOnMessageContext& that) { DoSwap(Message, that.Message); Ident.Swap(that.Ident); DoSwap(Session, that.Session); } - + TNetAddr GetPeerAddrNetAddr() const; bool IsConnectionAlive() const; }; - + struct IBusServerHandler: public IBusErrorHandler { virtual void OnMessage(TOnMessageContext& onMessage) = 0; /// called when reply has been sent from destination virtual void OnSent(TAutoPtr<TBusMessage> pMessage); }; - + struct IBusClientHandler: public IBusErrorHandler { /// called on source when reply arrives from destination virtual void OnReply(TAutoPtr<TBusMessage> pMessage, TAutoPtr<TBusMessage> pReply) = 0; @@ -131,5 +131,5 @@ namespace NBus { virtual void OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage); virtual void OnClientConnectionEvent(const TClientConnectionEvent&); }; - + } diff --git a/library/cpp/messagebus/handler_impl.h b/library/cpp/messagebus/handler_impl.h index f260767b47..6593f04cc3 100644 --- a/library/cpp/messagebus/handler_impl.h +++ b/library/cpp/messagebus/handler_impl.h @@ -1,23 +1,23 @@ -#pragma once - +#pragma once + #include "handler.h" #include "local_flags.h" -#include "session.h" - -namespace NBus { +#include "session.h" + +namespace NBus { template <typename U /* <: TBusMessage */> EMessageStatus TOnMessageContext::SendReplyAutoPtr(TAutoPtr<U>& response) { return Session->SendReplyAutoPtr(Ident, response); } - + inline EMessageStatus TOnMessageContext::SendReplyMove(TBusMessageAutoPtr response) { return SendReplyAutoPtr(response); } - + inline void TOnMessageContext::AckMessage(TBusIdentity& ident) { Y_VERIFY(Ident.LocalFlags == NPrivate::MESSAGE_IN_WORK); Y_VERIFY(ident.LocalFlags == 0); Ident.Swap(ident); } - -} + +} diff --git a/library/cpp/messagebus/hash.h b/library/cpp/messagebus/hash.h index bc3ad0df62..cc1b136a86 100644 --- a/library/cpp/messagebus/hash.h +++ b/library/cpp/messagebus/hash.h @@ -1,19 +1,19 @@ -#pragma once - -#include <util/str_stl.h> -#include <util/digest/numeric.h> - +#pragma once + +#include <util/str_stl.h> +#include <util/digest/numeric.h> + namespace NBus { namespace NPrivate { template <typename T> size_t Hash(const T& val) { return THash<T>()(val); } - + template <typename T, typename U> size_t HashValues(const T& a, const U& b) { return CombineHashes(Hash(a), Hash(b)); } - - } + + } } diff --git a/library/cpp/messagebus/key_value_printer.cpp b/library/cpp/messagebus/key_value_printer.cpp index 93851eff3a..c8592145c7 100644 --- a/library/cpp/messagebus/key_value_printer.cpp +++ b/library/cpp/messagebus/key_value_printer.cpp @@ -1,46 +1,46 @@ #include "key_value_printer.h" -#include <util/stream/format.h> - +#include <util/stream/format.h> + TKeyValuePrinter::TKeyValuePrinter(const TString& sep) - : Sep(sep) + : Sep(sep) { } - + TKeyValuePrinter::~TKeyValuePrinter() { } - + void TKeyValuePrinter::AddRowImpl(const TString& key, const TString& value, bool alignLeft) { - Keys.push_back(key); - Values.push_back(value); - AlignLefts.push_back(alignLeft); -} - + Keys.push_back(key); + Values.push_back(value); + AlignLefts.push_back(alignLeft); +} + TString TKeyValuePrinter::PrintToString() const { - if (Keys.empty()) { + if (Keys.empty()) { return TString(); - } - - size_t keyWidth = 0; - size_t valueWidth = 0; - - for (size_t i = 0; i < Keys.size(); ++i) { - keyWidth = Max(keyWidth, Keys.at(i).size()); - valueWidth = Max(valueWidth, Values.at(i).size()); - } - - TStringStream ss; - - for (size_t i = 0; i < Keys.size(); ++i) { - ss << RightPad(Keys.at(i), keyWidth); - ss << Sep; - if (AlignLefts.at(i)) { - ss << Values.at(i); - } else { - ss << LeftPad(Values.at(i), valueWidth); - } - ss << Endl; - } - - return ss.Str(); -} + } + + size_t keyWidth = 0; + size_t valueWidth = 0; + + for (size_t i = 0; i < Keys.size(); ++i) { + keyWidth = Max(keyWidth, Keys.at(i).size()); + valueWidth = Max(valueWidth, Values.at(i).size()); + } + + TStringStream ss; + + for (size_t i = 0; i < Keys.size(); ++i) { + ss << RightPad(Keys.at(i), keyWidth); + ss << Sep; + if (AlignLefts.at(i)) { + ss << Values.at(i); + } else { + ss << LeftPad(Values.at(i), valueWidth); + } + ss << Endl; + } + + return ss.Str(); +} diff --git a/library/cpp/messagebus/key_value_printer.h b/library/cpp/messagebus/key_value_printer.h index 34c6ec36c1..bca1fde50e 100644 --- a/library/cpp/messagebus/key_value_printer.h +++ b/library/cpp/messagebus/key_value_printer.h @@ -1,28 +1,28 @@ -#pragma once - +#pragma once + #include <util/generic/string.h> -#include <util/generic/typetraits.h> +#include <util/generic/typetraits.h> #include <util/generic/vector.h> #include <util/string/cast.h> - -class TKeyValuePrinter { -private: + +class TKeyValuePrinter { +private: TString Sep; TVector<TString> Keys; TVector<TString> Values; TVector<bool> AlignLefts; -public: +public: TKeyValuePrinter(const TString& sep = TString(": ")); - ~TKeyValuePrinter(); - - template <typename TKey, typename TValue> + ~TKeyValuePrinter(); + + template <typename TKey, typename TValue> void AddRow(const TKey& key, const TValue& value, bool leftAlign = !std::is_integral<TValue>::value) { - return AddRowImpl(ToString(key), ToString(value), leftAlign); - } - + return AddRowImpl(ToString(key), ToString(value), leftAlign); + } + TString PrintToString() const; - -private: + +private: void AddRowImpl(const TString& key, const TString& value, bool leftAlign); -}; +}; diff --git a/library/cpp/messagebus/latch.h b/library/cpp/messagebus/latch.h index d40aef2719..373f4c0e13 100644 --- a/library/cpp/messagebus/latch.h +++ b/library/cpp/messagebus/latch.h @@ -1,53 +1,53 @@ -#pragma once - +#pragma once + #include <util/system/condvar.h> -#include <util/system/mutex.h> - -class TLatch { -private: - // 0 for unlocked, 1 for locked - TAtomic Locked; - TMutex Mutex; - TCondVar CondVar; - -public: +#include <util/system/mutex.h> + +class TLatch { +private: + // 0 for unlocked, 1 for locked + TAtomic Locked; + TMutex Mutex; + TCondVar CondVar; + +public: TLatch() : Locked(0) { } - - void Wait() { - // optimistic path - if (AtomicGet(Locked) == 0) { - return; - } - - TGuard<TMutex> guard(Mutex); + + void Wait() { + // optimistic path + if (AtomicGet(Locked) == 0) { + return; + } + + TGuard<TMutex> guard(Mutex); while (AtomicGet(Locked) == 1) { - CondVar.WaitI(Mutex); - } - } - - bool TryWait() { - return AtomicGet(Locked) == 0; - } - - void Unlock() { - // optimistic path - if (AtomicGet(Locked) == 0) { - return; - } - - TGuard<TMutex> guard(Mutex); + CondVar.WaitI(Mutex); + } + } + + bool TryWait() { + return AtomicGet(Locked) == 0; + } + + void Unlock() { + // optimistic path + if (AtomicGet(Locked) == 0) { + return; + } + + TGuard<TMutex> guard(Mutex); AtomicSet(Locked, 0); - CondVar.BroadCast(); - } - - void Lock() { - AtomicSet(Locked, 1); - } - - bool IsLocked() { - return AtomicGet(Locked); - } -}; + CondVar.BroadCast(); + } + + void Lock() { + AtomicSet(Locked, 1); + } + + bool IsLocked() { + return AtomicGet(Locked); + } +}; diff --git a/library/cpp/messagebus/latch_ut.cpp b/library/cpp/messagebus/latch_ut.cpp index c1735b3339..bfab04f527 100644 --- a/library/cpp/messagebus/latch_ut.cpp +++ b/library/cpp/messagebus/latch_ut.cpp @@ -1,20 +1,20 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "latch.h" - + +#include "latch.h" + Y_UNIT_TEST_SUITE(TLatch) { Y_UNIT_TEST(Simple) { - TLatch latch; - UNIT_ASSERT(latch.TryWait()); - latch.Lock(); - UNIT_ASSERT(!latch.TryWait()); - latch.Lock(); - latch.Lock(); - UNIT_ASSERT(!latch.TryWait()); - latch.Unlock(); - UNIT_ASSERT(latch.TryWait()); - latch.Unlock(); - latch.Unlock(); - UNIT_ASSERT(latch.TryWait()); - } -} + TLatch latch; + UNIT_ASSERT(latch.TryWait()); + latch.Lock(); + UNIT_ASSERT(!latch.TryWait()); + latch.Lock(); + latch.Lock(); + UNIT_ASSERT(!latch.TryWait()); + latch.Unlock(); + UNIT_ASSERT(latch.TryWait()); + latch.Unlock(); + latch.Unlock(); + UNIT_ASSERT(latch.TryWait()); + } +} diff --git a/library/cpp/messagebus/left_right_buffer.h b/library/cpp/messagebus/left_right_buffer.h index c76aa10db6..f937cefad0 100644 --- a/library/cpp/messagebus/left_right_buffer.h +++ b/library/cpp/messagebus/left_right_buffer.h @@ -1,78 +1,78 @@ -#pragma once - +#pragma once + #include <util/generic/buffer.h> #include <util/generic/noncopyable.h> -#include <util/system/yassert.h> - +#include <util/system/yassert.h> + namespace NBus { namespace NPrivate { class TLeftRightBuffer : TNonCopyable { private: TBuffer Buffer; size_t Left; - + void CheckInvariant() { Y_ASSERT(Left <= Buffer.Size()); } - + public: TLeftRightBuffer() : Left(0) { } - + TBuffer& GetBuffer() { return Buffer; } - + size_t Capacity() { return Buffer.Capacity(); } - + void Clear() { Buffer.Clear(); Left = 0; } - + void Reset() { Buffer.Reset(); Left = 0; } - + void Compact() { Buffer.ChopHead(Left); Left = 0; } - + char* LeftPos() { return Buffer.Data() + Left; } - + size_t LeftSize() { return Left; } - + void LeftProceed(size_t count) { Y_ASSERT(count <= Size()); Left += count; } - + size_t Size() { return Buffer.Size() - Left; } - + bool Empty() { return Size() == 0; } - + char* RightPos() { return Buffer.Data() + Buffer.Size(); } - + size_t Avail() { return Buffer.Avail(); } }; - - } + + } } diff --git a/library/cpp/messagebus/lfqueue_batch.h b/library/cpp/messagebus/lfqueue_batch.h index f3db73a3dd..8128d3154d 100644 --- a/library/cpp/messagebus/lfqueue_batch.h +++ b/library/cpp/messagebus/lfqueue_batch.h @@ -1,36 +1,36 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/actor/temp_tls_vector.h> -#include <util/generic/vector.h> -#include <util/thread/lfstack.h> - +#include <util/generic/vector.h> +#include <util/thread/lfstack.h> + template <typename T, template <typename, class> class TVectorType = TVector> -class TLockFreeQueueBatch { -private: +class TLockFreeQueueBatch { +private: TLockFreeStack<TVectorType<T, std::allocator<T>>*> Stack; -public: - bool IsEmpty() { - return Stack.IsEmpty(); - } - +public: + bool IsEmpty() { + return Stack.IsEmpty(); + } + void EnqueueAll(TAutoPtr<TVectorType<T, std::allocator<T>>> vec) { - Stack.Enqueue(vec.Release()); - } - + Stack.Enqueue(vec.Release()); + } + void DequeueAllSingleConsumer(TVectorType<T, std::allocator<T>>* r) { TTempTlsVector<TVectorType<T, std::allocator<T>>*> vs; - Stack.DequeueAllSingleConsumer(vs.GetVector()); - + Stack.DequeueAllSingleConsumer(vs.GetVector()); + for (typename TVector<TVectorType<T, std::allocator<T>>*>::reverse_iterator i = vs.GetVector()->rbegin(); i != vs.GetVector()->rend(); ++i) { - if (i == vs.GetVector()->rend()) { - r->swap(**i); - } else { - r->insert(r->end(), (*i)->begin(), (*i)->end()); - } - delete *i; - } - } -}; + if (i == vs.GetVector()->rend()) { + r->swap(**i); + } else { + r->insert(r->end(), (*i)->begin(), (*i)->end()); + } + delete *i; + } + } +}; diff --git a/library/cpp/messagebus/lfqueue_batch_ut.cpp b/library/cpp/messagebus/lfqueue_batch_ut.cpp index 101f4cd932..f80434c0d4 100644 --- a/library/cpp/messagebus/lfqueue_batch_ut.cpp +++ b/library/cpp/messagebus/lfqueue_batch_ut.cpp @@ -1,56 +1,56 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "lfqueue_batch.h" - + +#include "lfqueue_batch.h" + Y_UNIT_TEST_SUITE(TLockFreeQueueBatch) { Y_UNIT_TEST(Order1) { - TLockFreeQueueBatch<unsigned> q; - { + TLockFreeQueueBatch<unsigned> q; + { TAutoPtr<TVector<unsigned>> v(new TVector<unsigned>); - v->push_back(0); - v->push_back(1); - q.EnqueueAll(v); - } - + v->push_back(0); + v->push_back(1); + q.EnqueueAll(v); + } + TVector<unsigned> r; - q.DequeueAllSingleConsumer(&r); - - UNIT_ASSERT_VALUES_EQUAL(2u, r.size()); - for (unsigned i = 0; i < 2; ++i) { - UNIT_ASSERT_VALUES_EQUAL(i, r[i]); - } - - r.clear(); - q.DequeueAllSingleConsumer(&r); - UNIT_ASSERT_VALUES_EQUAL(0u, r.size()); - } - + q.DequeueAllSingleConsumer(&r); + + UNIT_ASSERT_VALUES_EQUAL(2u, r.size()); + for (unsigned i = 0; i < 2; ++i) { + UNIT_ASSERT_VALUES_EQUAL(i, r[i]); + } + + r.clear(); + q.DequeueAllSingleConsumer(&r); + UNIT_ASSERT_VALUES_EQUAL(0u, r.size()); + } + Y_UNIT_TEST(Order2) { - TLockFreeQueueBatch<unsigned> q; - { + TLockFreeQueueBatch<unsigned> q; + { TAutoPtr<TVector<unsigned>> v(new TVector<unsigned>); - v->push_back(0); - v->push_back(1); - q.EnqueueAll(v); - } - { + v->push_back(0); + v->push_back(1); + q.EnqueueAll(v); + } + { TAutoPtr<TVector<unsigned>> v(new TVector<unsigned>); - v->push_back(2); - v->push_back(3); - v->push_back(4); - q.EnqueueAll(v); - } - + v->push_back(2); + v->push_back(3); + v->push_back(4); + q.EnqueueAll(v); + } + TVector<unsigned> r; - q.DequeueAllSingleConsumer(&r); - - UNIT_ASSERT_VALUES_EQUAL(5u, r.size()); - for (unsigned i = 0; i < 5; ++i) { - UNIT_ASSERT_VALUES_EQUAL(i, r[i]); - } - - r.clear(); - q.DequeueAllSingleConsumer(&r); - UNIT_ASSERT_VALUES_EQUAL(0u, r.size()); - } -} + q.DequeueAllSingleConsumer(&r); + + UNIT_ASSERT_VALUES_EQUAL(5u, r.size()); + for (unsigned i = 0; i < 5; ++i) { + UNIT_ASSERT_VALUES_EQUAL(i, r[i]); + } + + r.clear(); + q.DequeueAllSingleConsumer(&r); + UNIT_ASSERT_VALUES_EQUAL(0u, r.size()); + } +} diff --git a/library/cpp/messagebus/local_flags.cpp b/library/cpp/messagebus/local_flags.cpp index aac65590ed..877e533f76 100644 --- a/library/cpp/messagebus/local_flags.cpp +++ b/library/cpp/messagebus/local_flags.cpp @@ -1,19 +1,19 @@ #include "local_flags.h" #include <util/stream/str.h> -#include <util/string/printf.h> - -using namespace NBus; -using namespace NBus::NPrivate; - +#include <util/string/printf.h> + +using namespace NBus; +using namespace NBus::NPrivate; + TString NBus::NPrivate::LocalFlagSetToString(ui32 flags0) { - if (flags0 == 0) { - return "0"; - } - - ui32 flags = flags0; - - TStringStream ss; + if (flags0 == 0) { + return "0"; + } + + ui32 flags = flags0; + + TStringStream ss; #define P(name, value, ...) \ do \ if (flags & value) { \ @@ -23,10 +23,10 @@ TString NBus::NPrivate::LocalFlagSetToString(ui32 flags0) { ss << #name; \ flags &= ~name; \ } \ - while (false); - MESSAGE_LOCAL_FLAGS_MAP(P) - if (flags != 0) { - return Sprintf("0x%x", unsigned(flags0)); - } - return ss.Str(); -} + while (false); + MESSAGE_LOCAL_FLAGS_MAP(P) + if (flags != 0) { + return Sprintf("0x%x", unsigned(flags0)); + } + return ss.Str(); +} diff --git a/library/cpp/messagebus/local_flags.h b/library/cpp/messagebus/local_flags.h index fe1f157bdf..f589283188 100644 --- a/library/cpp/messagebus/local_flags.h +++ b/library/cpp/messagebus/local_flags.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> #include <util/generic/string.h> -#include <util/stream/output.h> - +#include <util/stream/output.h> + namespace NBus { namespace NPrivate { #define MESSAGE_LOCAL_FLAGS_MAP(XX) \ @@ -14,13 +14,13 @@ namespace NBus { XX(MESSAGE_REPLY_IS_BEGING_SENT, 0x0008) \ XX(MESSAGE_ONE_WAY_INTERNAL, 0x0010) \ /**/ - + enum EMessageLocalFlags { MESSAGE_LOCAL_FLAGS_MAP(ENUM_VALUE_GEN) }; - + ENUM_TO_STRING(EMessageLocalFlags, MESSAGE_LOCAL_FLAGS_MAP) - + TString LocalFlagSetToString(ui32); } } diff --git a/library/cpp/messagebus/local_flags_ut.cpp b/library/cpp/messagebus/local_flags_ut.cpp index 5fdb483db6..189d73eb0f 100644 --- a/library/cpp/messagebus/local_flags_ut.cpp +++ b/library/cpp/messagebus/local_flags_ut.cpp @@ -1,18 +1,18 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "local_flags.h" - -using namespace NBus; -using namespace NBus::NPrivate; - + +#include "local_flags.h" + +using namespace NBus; +using namespace NBus::NPrivate; + Y_UNIT_TEST_SUITE(EMessageLocalFlags) { Y_UNIT_TEST(TestLocalFlagSetToString) { - UNIT_ASSERT_VALUES_EQUAL("0", LocalFlagSetToString(0)); - UNIT_ASSERT_VALUES_EQUAL("MESSAGE_REPLY_INTERNAL", + UNIT_ASSERT_VALUES_EQUAL("0", LocalFlagSetToString(0)); + UNIT_ASSERT_VALUES_EQUAL("MESSAGE_REPLY_INTERNAL", LocalFlagSetToString(MESSAGE_REPLY_INTERNAL)); - UNIT_ASSERT_VALUES_EQUAL("MESSAGE_IN_WORK|MESSAGE_IN_FLIGHT_ON_CLIENT", + UNIT_ASSERT_VALUES_EQUAL("MESSAGE_IN_WORK|MESSAGE_IN_FLIGHT_ON_CLIENT", LocalFlagSetToString(MESSAGE_IN_WORK | MESSAGE_IN_FLIGHT_ON_CLIENT)); - UNIT_ASSERT_VALUES_EQUAL("0xff3456", + UNIT_ASSERT_VALUES_EQUAL("0xff3456", LocalFlagSetToString(0xff3456)); - } -} + } +} diff --git a/library/cpp/messagebus/local_tasks.h b/library/cpp/messagebus/local_tasks.h index 4a07bf95f5..d8e801a457 100644 --- a/library/cpp/messagebus/local_tasks.h +++ b/library/cpp/messagebus/local_tasks.h @@ -1,23 +1,23 @@ -#pragma once - +#pragma once + #include <util/system/atomic.h> -class TLocalTasks { -private: +class TLocalTasks { +private: TAtomic GotTasks; - -public: - TLocalTasks() + +public: + TLocalTasks() : GotTasks(0) { } - - void AddTask() { + + void AddTask() { AtomicSet(GotTasks, 1); - } - - bool FetchTask() { + } + + bool FetchTask() { bool gotTasks = AtomicCas(&GotTasks, 0, 1); - return gotTasks; - } -}; + return gotTasks; + } +}; diff --git a/library/cpp/messagebus/locator.cpp b/library/cpp/messagebus/locator.cpp index 31cdf25805..e38a35c426 100644 --- a/library/cpp/messagebus/locator.cpp +++ b/library/cpp/messagebus/locator.cpp @@ -2,7 +2,7 @@ /// \file /// \brief Implementation of locator service -#include "locator.h" +#include "locator.h" #include "ybus.h" @@ -121,7 +121,7 @@ namespace NBus { , End(end) , Addr(addr) { - } + } bool TBusLocator::IsLocal(const TNetAddr& addr) { for (const auto& myInterface : MyInterfaces) { @@ -151,7 +151,7 @@ namespace NBus { int TBusLocator::RegisterBreak(TServiceId serviceId, const TBusKey start, const TNetAddr& addr) { TItems::const_iterator it = Items.lower_bound(TItem(serviceId, 0, start, addr)); TItems::const_iterator service_it = - Items.lower_bound(TItem(serviceId, 0, 0, TNetAddr())); + Items.lower_bound(TItem(serviceId, 0, 0, TNetAddr())); THolder<TItem> left; THolder<TItem> right; @@ -220,11 +220,11 @@ namespace NBus { } keyItem = item.Start - 1; } while (it != Items.begin()); - + if (first != Items.end() && first->Start != 0) { TItem item(serviceId, YBUS_KEYMIN, first->Start - 1, first->Addr); Items.insert(item); - } + } NormalizeBreaks(serviceId); return deleted; @@ -245,7 +245,7 @@ namespace NBus { TItem& beg = const_cast<TItem&>(*first); beg.Addr = last->Addr; - } + } } int TBusLocator::LocateAll(TBusService service, TBusKey key, TVector<TNetAddr>& addrs) { @@ -307,7 +307,7 @@ namespace NBus { } port = GetAddrPort(item.Addr); } - } + } return port; } @@ -326,7 +326,7 @@ namespace NBus { if (IsLocal(item.Addr)) { addrs.push_back(item.Addr); } - } + } if (addrs.size() == 0) { return -1; diff --git a/library/cpp/messagebus/locator.h b/library/cpp/messagebus/locator.h index b90a7aceb4..f8556a3fce 100644 --- a/library/cpp/messagebus/locator.h +++ b/library/cpp/messagebus/locator.h @@ -1,18 +1,18 @@ -#pragma once - +#pragma once + #include "defs.h" -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/generic/map.h> #include <util/generic/set.h> #include <util/generic/string.h> #include <util/network/interface.h> #include <util/system/mutex.h> - -namespace NBus { + +namespace NBus { /////////////////////////////////////////////// /// \brief Client interface to locator service - + /// This interface abstracts clustering/location service that /// allows clients find servers (address, port) using "name" and "key". /// The instance lives in TBusMessageQueue-object, but can be shared by different queues. @@ -22,72 +22,72 @@ namespace NBus { typedef TSet<TString> TServiceIdSet; TServiceIdSet ServiceIdSet; TServiceId GetServiceId(const char* name); - + typedef TMap<TNetAddr, TString> THostAddrMap; THostAddrMap HostAddrMap; - + TNetworkInterfaceList MyInterfaces; - + struct TItem { TServiceId ServiceId; TBusKey Start; TBusKey End; TNetAddr Addr; - + bool operator<(const TItem& y) const; - + bool operator==(const TItem& y) const; - + TItem(TServiceId serviceId, TBusKey start, TBusKey end, const TNetAddr& addr); }; typedef TMultiSet<TItem> TItems; TItems Items; TMutex Lock; - + int RegisterBreak(TServiceId serviceId, const TBusKey start, const TNetAddr& addr); int UnregisterBreak(TServiceId serviceId, const TNetAddr& addr); - + void NormalizeBreaks(TServiceId serviceId); - + private: int Register(TBusService service, TBusKey start, TBusKey end, const TNetAddr& addr); - + public: /// creates instance that obtains location table from locator server (not implemented) TBusLocator(); - + /// returns true if this address is on the same node for YBUS_KEYLOCAL bool IsLocal(const TNetAddr& addr); - + /// returns first address for service and key int Locate(TBusService service, TBusKey key, TNetAddr* addr); - + /// returns all addresses mathing service and key int LocateAll(TBusService service, TBusKey key, TVector<TNetAddr>& addrs); - + /// returns actual host name for service and key int LocateHost(TBusService service, TBusKey key, TString* host, int* port, bool* isLocal = nullptr); - + /// returns all key ranges for the given service int LocateKeys(TBusService service, TBusKeyVec& keys, bool onlyLocal = false); - + /// returns port on the local node for the service int GetLocalPort(TBusService service); - + /// returns addresses of the local node for the service int GetLocalAddresses(TBusService service, TVector<TNetAddr>& addrs); - + /// register service instance int Register(TBusService service, TBusKey start, TBusKey end, const TNetworkAddress& addr, EIpVersion requireVersion = EIP_VERSION_4, EIpVersion preferVersion = EIP_VERSION_ANY); /// @throws yexception int Register(TBusService service, const char* host, int port, TBusKey start = YBUS_KEYMIN, TBusKey end = YBUS_KEYMAX, EIpVersion requireVersion = EIP_VERSION_4, EIpVersion preferVersion = EIP_VERSION_ANY); - + /// unregister service instance int Unregister(TBusService service, TBusKey start, TBusKey end); - + int RegisterBreak(TBusService service, const TVector<TBusKey>& starts, const TNetAddr& addr); int UnregisterBreak(TBusService service, const TNetAddr& addr); }; - + } diff --git a/library/cpp/messagebus/mb_lwtrace.cpp b/library/cpp/messagebus/mb_lwtrace.cpp index 1a1cc3997e..c54cd5ab71 100644 --- a/library/cpp/messagebus/mb_lwtrace.cpp +++ b/library/cpp/messagebus/mb_lwtrace.cpp @@ -1,12 +1,12 @@ #include "mb_lwtrace.h" - + #include <library/cpp/lwtrace/all.h> - + #include <util/generic/singleton.h> - -LWTRACE_DEFINE_PROVIDER(LWTRACE_MESSAGEBUS_PROVIDER) - -void NBus::InitBusLwtrace() { - // Function is nop, and needed only to make sure TBusLwtraceInit loaded. - // It won't be necessary when pg@ implements GLOBAL in arc. -} + +LWTRACE_DEFINE_PROVIDER(LWTRACE_MESSAGEBUS_PROVIDER) + +void NBus::InitBusLwtrace() { + // Function is nop, and needed only to make sure TBusLwtraceInit loaded. + // It won't be necessary when pg@ implements GLOBAL in arc. +} diff --git a/library/cpp/messagebus/mb_lwtrace.h b/library/cpp/messagebus/mb_lwtrace.h index 032e1989ea..e62728b265 100644 --- a/library/cpp/messagebus/mb_lwtrace.h +++ b/library/cpp/messagebus/mb_lwtrace.h @@ -1,7 +1,7 @@ -#pragma once - +#pragma once + #include <library/cpp/lwtrace/all.h> - + #include <util/generic/string.h> #define LWTRACE_MESSAGEBUS_PROVIDER(PROBE, EVENT, GROUPS, TYPES, NAMES) \ @@ -10,10 +10,10 @@ PROBE(Accepted, GROUPS("MessagebusRare"), TYPES(TString), NAMES("address")) \ PROBE(Disconnected, GROUPS("MessagebusRare"), TYPES(TString), NAMES("address")) \ PROBE(Read, GROUPS(), TYPES(ui32), NAMES("size")) \ - /**/ - -LWTRACE_DECLARE_PROVIDER(LWTRACE_MESSAGEBUS_PROVIDER) - -namespace NBus { - void InitBusLwtrace(); -} + /**/ + +LWTRACE_DECLARE_PROVIDER(LWTRACE_MESSAGEBUS_PROVIDER) + +namespace NBus { + void InitBusLwtrace(); +} diff --git a/library/cpp/messagebus/memory.h b/library/cpp/messagebus/memory.h index e7bfc1827d..b2c0544491 100644 --- a/library/cpp/messagebus/memory.h +++ b/library/cpp/messagebus/memory.h @@ -1,42 +1,42 @@ -#pragma once - -#ifndef CACHE_LINE_SIZE -#define CACHE_LINE_SIZE 64 -#endif - -#define CONCAT(a, b) a##b -#define LABEL(a) CONCAT(UniqueName_, a) -#define UNIQUE_NAME LABEL(__LINE__) - -#define CACHE_LINE_PADDING char UNIQUE_NAME[CACHE_LINE_SIZE]; - -static inline void* MallocAligned(size_t size, size_t alignment) { +#pragma once + +#ifndef CACHE_LINE_SIZE +#define CACHE_LINE_SIZE 64 +#endif + +#define CONCAT(a, b) a##b +#define LABEL(a) CONCAT(UniqueName_, a) +#define UNIQUE_NAME LABEL(__LINE__) + +#define CACHE_LINE_PADDING char UNIQUE_NAME[CACHE_LINE_SIZE]; + +static inline void* MallocAligned(size_t size, size_t alignment) { void** ptr = (void**)malloc(size + alignment + sizeof(size_t*)); - if (!ptr) { + if (!ptr) { return nullptr; - } - - size_t mask = ~(alignment - 1); - intptr_t roundedDown = intptr_t(ptr) & mask; + } + + size_t mask = ~(alignment - 1); + intptr_t roundedDown = intptr_t(ptr) & mask; void** alignedPtr = (void**)(roundedDown + alignment); - alignedPtr[-1] = ptr; - return alignedPtr; -} - -static inline void FreeAligned(void* ptr) { - if (!ptr) { - return; - } - + alignedPtr[-1] = ptr; + return alignedPtr; +} + +static inline void FreeAligned(void* ptr) { + if (!ptr) { + return; + } + void** typedPtr = (void**)ptr; void* originalPtr = typedPtr[-1]; - free(originalPtr); -} - -static inline void* MallocCacheAligned(size_t size) { - return MallocAligned(size, CACHE_LINE_SIZE); -} - -static inline void FreeCacheAligned(void* ptr) { - return FreeAligned(ptr); -} + free(originalPtr); +} + +static inline void* MallocCacheAligned(size_t size) { + return MallocAligned(size, CACHE_LINE_SIZE); +} + +static inline void FreeCacheAligned(void* ptr) { + return FreeAligned(ptr); +} diff --git a/library/cpp/messagebus/memory_ut.cpp b/library/cpp/messagebus/memory_ut.cpp index 036eb43b4d..00654f28a1 100644 --- a/library/cpp/messagebus/memory_ut.cpp +++ b/library/cpp/messagebus/memory_ut.cpp @@ -1,13 +1,13 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "memory.h" - + +#include "memory.h" + Y_UNIT_TEST_SUITE(MallocAligned) { Y_UNIT_TEST(Test) { - for (size_t size = 0; size < 1000; ++size) { - void* ptr = MallocAligned(size, 128); - UNIT_ASSERT(uintptr_t(ptr) % 128 == 0); - FreeAligned(ptr); - } - } -} + for (size_t size = 0; size < 1000; ++size) { + void* ptr = MallocAligned(size, 128); + UNIT_ASSERT(uintptr_t(ptr) % 128 == 0); + FreeAligned(ptr); + } + } +} diff --git a/library/cpp/messagebus/message.cpp b/library/cpp/messagebus/message.cpp index eca319326e..bfa7ed8e9b 100644 --- a/library/cpp/messagebus/message.cpp +++ b/library/cpp/messagebus/message.cpp @@ -1,14 +1,14 @@ #include "remote_server_connection.h" #include "ybus.h" - + #include <util/random/random.h> #include <util/string/printf.h> #include <util/system/atomic.h> #include <string.h> -using namespace NBus; - +using namespace NBus; + namespace NBus { using namespace NBus::NPrivate; @@ -19,40 +19,40 @@ namespace NBus { , LocalFlags(0) { } - + TBusIdentity::~TBusIdentity() { - // TODO: print local flags -#ifndef NDEBUG + // TODO: print local flags +#ifndef NDEBUG Y_VERIFY(LocalFlags == 0, "local flags must be zero at this point; message type is %s", MessageType.value_or("unknown").c_str()); -#else +#else Y_VERIFY(LocalFlags == 0, "local flags must be zero at this point"); -#endif +#endif } - + TNetAddr TBusIdentity::GetNetAddr() const { if (!!Connection) { return Connection->GetAddr(); } else { Y_FAIL(); } - } - + } + void TBusIdentity::Pack(char* dest) { memcpy(dest, this, sizeof(TBusIdentity)); LocalFlags = 0; - + // prevent decref new (&Connection) TIntrusivePtr<TRemoteServerConnection>; } - + void TBusIdentity::Unpack(const char* src) { Y_VERIFY(LocalFlags == 0); Y_VERIFY(!Connection); - + memcpy(this, src, sizeof(TBusIdentity)); } - + void TBusHeader::GenerateId() { for (;;) { Id = RandomNumber<TBusKey>(); @@ -61,7 +61,7 @@ namespace NBus { return; } } - + TBusMessage::TBusMessage(ui16 type, int approxsize) //: TCtr("BusMessage") : TRefCounted<TBusMessage, TAtomicCounter, TDelete>(1) @@ -72,7 +72,7 @@ namespace NBus { Y_UNUSED(approxsize); GetHeader()->Type = type; DoReset(); - } + } TBusMessage::TBusMessage(ECreateUninitialized) //: TCtr("BusMessage") @@ -87,12 +87,12 @@ namespace NBus { } TBusMessage::~TBusMessage() { -#ifndef NDEBUG +#ifndef NDEBUG Y_VERIFY(GetHeader()->Id != YBUS_KEYINVALID, "must not be invalid key, message type: %d, ", int(Type)); GetHeader()->Id = YBUS_KEYINVALID; Data = (void*)17; CheckClean(); -#endif +#endif } void TBusMessage::DoReset() { @@ -102,20 +102,20 @@ namespace NBus { GetHeader()->GenerateId(); GetHeader()->SetVersionInternal(); } - + void TBusMessage::Reset() { CheckClean(); DoReset(); } - + void TBusMessage::CheckClean() const { if (Y_UNLIKELY(LocalFlags != 0)) { TString describe = Describe(); TString localFlags = LocalFlagSetToString(LocalFlags); Y_FAIL("message local flags must be zero, got: %s, message: %s", localFlags.data(), describe.data()); } - } - + } + /////////////////////////////////////////////////////// /// \brief Unpacks header from network order @@ -181,18 +181,18 @@ namespace NBus { ss << " conn=" << Connection->GetAddr(); } ss - << " flags=" << Flags - << " local-flags=" << LocalFlags -#ifndef NDEBUG + << " flags=" << Flags + << " local-flags=" << LocalFlags +#ifndef NDEBUG << " msg-type= " << MessageType.value_or("unknown").c_str() -#endif - ; +#endif + ; return ss.Str(); } - -} - -template <> + +} + +template <> void Out<TBusIdentity>(IOutputStream& os, TTypeTraits<TBusIdentity>::TFuncParam ident) { - os << ident.ToString(); -} + os << ident.ToString(); +} diff --git a/library/cpp/messagebus/message.h b/library/cpp/messagebus/message.h index 6fce2b1fac..005ca10c65 100644 --- a/library/cpp/messagebus/message.h +++ b/library/cpp/messagebus/message.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include "base.h" #include "local_flags.h" #include "message_status.h" @@ -8,16 +8,16 @@ #include <util/generic/array_ref.h> #include <util/generic/noncopyable.h> -#include <util/generic/ptr.h> +#include <util/generic/ptr.h> #include <util/generic/string.h> #include <util/system/defaults.h> #include <util/system/type_name.h> -#include <util/system/yassert.h> - +#include <util/system/yassert.h> + #include <optional> #include <typeinfo> -namespace NBus { +namespace NBus { /////////////////////////////////////////////////////////////////// /// \brief Structure to preserve identity from message to reply struct TBusIdentity : TNonCopyable { @@ -25,33 +25,33 @@ namespace NBus { friend class NPrivate::TRemoteServerSession; friend struct NPrivate::TClientRequestImpl; friend class TOnMessageContext; - + // TODO: make private TBusKey MessageId; - + private: ui32 Size; TIntrusivePtr<NPrivate::TRemoteServerConnection> Connection; ui16 Flags; ui32 LocalFlags; TInstant RecvTime; - -#ifndef NDEBUG + +#ifndef NDEBUG std::optional<TString> MessageType; -#endif - +#endif + private: // TODO: drop TNetAddr GetNetAddr() const; - + public: void Pack(char* dest); void Unpack(const char* src); - + bool IsInWork() const { return LocalFlags & NPrivate::MESSAGE_IN_WORK; } - + // for internal use only void BeginWork() { SetInWork(true); @@ -64,7 +64,7 @@ namespace NBus { TBusIdentity(); ~TBusIdentity(); - + void Swap(TBusIdentity& that) { DoSwap(MessageId, that.MessageId); DoSwap(Size, that.Size); @@ -72,13 +72,13 @@ namespace NBus { DoSwap(Flags, that.Flags); DoSwap(LocalFlags, that.LocalFlags); DoSwap(RecvTime, that.RecvTime); -#ifndef NDEBUG +#ifndef NDEBUG DoSwap(MessageType, that.MessageType); -#endif +#endif } - + TString ToString() const; - + private: void SetInWork(bool inWork) { if (LocalFlags == 0 && inWork) { @@ -89,20 +89,20 @@ namespace NBus { Y_FAIL("impossible combination of flag and parameter: %s %d", inWork ? "true" : "false", unsigned(LocalFlags)); } - } - + } + void SetMessageType(const std::type_info& messageTypeInfo) { -#ifndef NDEBUG +#ifndef NDEBUG Y_VERIFY(!MessageType, "state check"); MessageType = TypeName(messageTypeInfo); -#else +#else Y_UNUSED(messageTypeInfo); -#endif +#endif } }; - + static const size_t BUS_IDENTITY_PACKED_SIZE = sizeof(TBusIdentity); - + /////////////////////////////////////////////////////////////// /// \brief Message flags in TBusHeader.Flags enum EMessageFlags { @@ -110,59 +110,59 @@ namespace NBus { MESSAGE_COMPRESS_RESPONSE = 0x4000, ///< message prefers compressed response MESSAGE_VERSION_INTERNAL = 0x00F0, ///< these bits are used as version }; - -////////////////////////////////////////////////////////// -/// \brief Message header present in all message send and received - -/// This header is send into the wire. -/// \todo fix for low/high end, 32/64bit some day -#pragma pack(1) + +////////////////////////////////////////////////////////// +/// \brief Message header present in all message send and received + +/// This header is send into the wire. +/// \todo fix for low/high end, 32/64bit some day +#pragma pack(1) struct TBusHeader { friend class TBusMessage; - + TBusKey Id = 0; ///< unique message ID ui32 Size = 0; ///< total size of the message TBusInstant SendTime = 0; ///< time the message was sent ui16 FlagsInternal = 0; ///< TRACE is one of the flags ui16 Type = 0; ///< to be used by TBusProtocol - + int GetVersionInternal() { return (FlagsInternal & MESSAGE_VERSION_INTERNAL) >> 4; } void SetVersionInternal(unsigned ver = YBUS_VERSION) { FlagsInternal |= (ver << 4); } - + public: TBusHeader() { } TBusHeader(TArrayRef<const char> data) { ReadHeader(data); } - + private: /// function for serialization/deserialization of the header /// returns number of bytes written/read int ReadHeader(TArrayRef<const char> data); - + void GenerateId(); }; -#pragma pack() - +#pragma pack() + #define TBUSMAX_MESSAGE 26 * 1024 * 1024 + sizeof(NBus::TBusHeader) ///< is't it enough? #define TBUSMIN_MESSAGE sizeof(NBus::TBusHeader) ///< can't be less then header - + inline bool IsVersionNegotiation(const NBus::TBusHeader& header) { return header.Id == 0 && header.Size == sizeof(TBusHeader); } ////////////////////////////////////////////////////////// /// \brief Base class for all messages passed in the system - + enum ECreateUninitialized { MESSAGE_CREATE_UNINITIALIZED, }; - + class TBusMessage : protected TBusHeader, public TRefCounted<TBusMessage, TAtomicCounter, TDelete>, @@ -175,35 +175,35 @@ namespace NBus { friend class ::NBus::NPrivate::TRemoteClientConnection; friend class ::NBus::NPrivate::TRemoteServerConnection; friend struct ::NBus::NPrivate::TBusMessagePtrAndHeader; - + private: ui32 LocalFlags; - + /// connection identity for reply set by PushMessage() NPrivate::TBusSocketAddr ReplyTo; // server-side response only, hack ui32 RequestSize; - + TInstant RecvTime; - + public: /// constructor to create messages on sending end TBusMessage(ui16 type, int approxsize = sizeof(TBusHeader)); - + /// constructor with serialzed data to examine the header TBusMessage(ECreateUninitialized); - + // slow, for diagnostics only virtual TString Describe() const; - + // must be called if this message object needs to be reused void Reset(); - + void CheckClean() const; - + void SetCompressed(bool); void SetCompressedResponse(bool); - + private: bool IsCompressed() const { return FlagsInternal & MESSAGE_COMPRESS_INTERNAL; @@ -211,11 +211,11 @@ namespace NBus { bool IsCompressedResponse() const { return FlagsInternal & MESSAGE_COMPRESS_RESPONSE; } - + public: /// can have private data to destroy virtual ~TBusMessage(); - + /// returns header of the message TBusHeader* GetHeader() { return this; @@ -223,50 +223,50 @@ namespace NBus { const TBusHeader* GetHeader() const { return this; } - + /// helper to return type for protocol object to unpack object static ui16 GetType(TArrayRef<const char> data) { return TBusHeader(data).Type; } - + /// returns payload data static TArrayRef<const char> GetPayload(TArrayRef<const char> data) { return data.Slice(sizeof(TBusHeader)); } - + private: void DoReset(); - + /// serialize message identity to be used to construct reply message void GetIdentity(TBusIdentity& ident) const; - + /// set message identity from serialized form void SetIdentity(const TBusIdentity& ident); - + public: TNetAddr GetReplyTo() const { return ReplyTo.ToNetAddr(); } - + /// store of application specific data, never serialized into wire void* Data; }; - + class TBusMessageAutoPtr: public TAutoPtr<TBusMessage> { public: TBusMessageAutoPtr() { } - + TBusMessageAutoPtr(TBusMessage* message) : TAutoPtr<TBusMessage>(message) { } - + template <typename T1> TBusMessageAutoPtr(const TAutoPtr<T1>& that) : TAutoPtr<TBusMessage>(that.Release()) { } }; - + } diff --git a/library/cpp/messagebus/message_counter.cpp b/library/cpp/messagebus/message_counter.cpp index 93092a65a6..04d9343f6a 100644 --- a/library/cpp/messagebus/message_counter.cpp +++ b/library/cpp/messagebus/message_counter.cpp @@ -1,46 +1,46 @@ #include "message_counter.h" -#include <util/stream/str.h> - -using namespace NBus; -using namespace NBus::NPrivate; - -TMessageCounter::TMessageCounter() - : BytesData(0) - , BytesNetwork(0) - , Count(0) - , CountCompressed(0) - , CountCompressionRequests(0) +#include <util/stream/str.h> + +using namespace NBus; +using namespace NBus::NPrivate; + +TMessageCounter::TMessageCounter() + : BytesData(0) + , BytesNetwork(0) + , Count(0) + , CountCompressed(0) + , CountCompressionRequests(0) { } - -TMessageCounter& TMessageCounter::operator+=(const TMessageCounter& that) { - BytesData += that.BytesData; - BytesNetwork += that.BytesNetwork; - Count += that.Count; - CountCompressed += that.CountCompressed; - CountCompressionRequests += that.CountCompressionRequests; - return *this; -} - + +TMessageCounter& TMessageCounter::operator+=(const TMessageCounter& that) { + BytesData += that.BytesData; + BytesNetwork += that.BytesNetwork; + Count += that.Count; + CountCompressed += that.CountCompressed; + CountCompressionRequests += that.CountCompressionRequests; + return *this; +} + TString TMessageCounter::ToString(bool reader) const { - if (reader) { + if (reader) { Y_ASSERT(CountCompressionRequests == 0); - } - - TStringStream readValue; - readValue << Count; - if (CountCompressionRequests != 0 || CountCompressed != 0) { - readValue << " (" << CountCompressed << " compr"; - if (!reader) { - readValue << ", " << CountCompressionRequests << " compr reqs"; - } - readValue << ")"; - } - readValue << ", "; - readValue << BytesData << "b"; - if (BytesNetwork != BytesData) { - readValue << " (" << BytesNetwork << "b network)"; - } - return readValue.Str(); -} + } + + TStringStream readValue; + readValue << Count; + if (CountCompressionRequests != 0 || CountCompressed != 0) { + readValue << " (" << CountCompressed << " compr"; + if (!reader) { + readValue << ", " << CountCompressionRequests << " compr reqs"; + } + readValue << ")"; + } + readValue << ", "; + readValue << BytesData << "b"; + if (BytesNetwork != BytesData) { + readValue << " (" << BytesNetwork << "b network)"; + } + return readValue.Str(); +} diff --git a/library/cpp/messagebus/message_counter.h b/library/cpp/messagebus/message_counter.h index c0c72df5f3..e4be1180b0 100644 --- a/library/cpp/messagebus/message_counter.h +++ b/library/cpp/messagebus/message_counter.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <util/generic/string.h> #include <cstddef> @@ -12,7 +12,7 @@ namespace NBus { size_t Count; size_t CountCompressed; size_t CountCompressionRequests; // reader only - + void AddMessage(size_t bytesData, size_t bytesCompressed, bool Compressed, bool compressionRequested) { BytesData += bytesData; BytesNetwork += bytesCompressed; @@ -24,13 +24,13 @@ namespace NBus { CountCompressionRequests += 1; } } - + TMessageCounter& operator+=(const TMessageCounter& that); - + TString ToString(bool reader) const; - + TMessageCounter(); }; - } + } } diff --git a/library/cpp/messagebus/message_ptr_and_header.h b/library/cpp/messagebus/message_ptr_and_header.h index a28ec7b08b..9b4e2fd270 100644 --- a/library/cpp/messagebus/message_ptr_and_header.h +++ b/library/cpp/messagebus/message_ptr_and_header.h @@ -1,36 +1,36 @@ -#pragma once - +#pragma once + #include "message.h" -#include "nondestroying_holder.h" - +#include "nondestroying_holder.h" + #include <util/generic/noncopyable.h> #include <util/generic/utility.h> - + namespace NBus { namespace NPrivate { struct TBusMessagePtrAndHeader : TNonCopyable { TNonDestroyingHolder<TBusMessage> MessagePtr; TBusHeader Header; ui32 LocalFlags; - + TBusMessagePtrAndHeader() : LocalFlags() { } - + explicit TBusMessagePtrAndHeader(TBusMessage* messagePtr) : MessagePtr(messagePtr) , Header(*MessagePtr->GetHeader()) , LocalFlags(MessagePtr->LocalFlags) { } - + void Swap(TBusMessagePtrAndHeader& that) { DoSwap(MessagePtr, that.MessagePtr); DoSwap(Header, that.Header); DoSwap(LocalFlags, that.LocalFlags); } }; - + } } diff --git a/library/cpp/messagebus/message_status.cpp b/library/cpp/messagebus/message_status.cpp index c895e6a9df..41ad62b73f 100644 --- a/library/cpp/messagebus/message_status.cpp +++ b/library/cpp/messagebus/message_status.cpp @@ -1,13 +1,13 @@ -#include "message_status.h" - -using namespace NBus; - -const char* NBus::MessageStatusDescription(EMessageStatus messageStatus) { +#include "message_status.h" + +using namespace NBus; + +const char* NBus::MessageStatusDescription(EMessageStatus messageStatus) { #define MESSAGE_STATUS_DESCRIPTION_GEN(name, description, ...) \ if (messageStatus == name) \ return description; - - MESSAGE_STATUS_MAP(MESSAGE_STATUS_DESCRIPTION_GEN) - - return "Unknown"; -} + + MESSAGE_STATUS_MAP(MESSAGE_STATUS_DESCRIPTION_GEN) + + return "Unknown"; +} diff --git a/library/cpp/messagebus/message_status.h b/library/cpp/messagebus/message_status.h index d7611da4dc..e1878960b3 100644 --- a/library/cpp/messagebus/message_status.h +++ b/library/cpp/messagebus/message_status.h @@ -1,14 +1,14 @@ -#pragma once - +#pragma once + #include "codegen.h" #include "defs.h" #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> - -namespace NBus { -//////////////////////////////////////////////////////////////// -/// \brief Status of message communication - + +namespace NBus { +//////////////////////////////////////////////////////////////// +/// \brief Status of message communication + #define MESSAGE_STATUS_MAP(XX) \ XX(MESSAGE_OK, "OK") \ XX(MESSAGE_CONNECT_FAILED, "Connect failed") \ @@ -26,20 +26,20 @@ namespace NBus { XX(MESSAGE_SERVICE_TOOMANY, "Locator failed to resolve address") \ XX(MESSAGE_SHUTDOWN, "Failure because of either session or connection shutdown") \ XX(MESSAGE_DONT_ASK, "Internal error code used by modules") - + enum EMessageStatus { MESSAGE_STATUS_MAP(ENUM_VALUE_GEN_NO_VALUE) MESSAGE_STATUS_COUNT }; - + ENUM_TO_STRING(EMessageStatus, MESSAGE_STATUS_MAP) - + const char* MessageStatusDescription(EMessageStatus); - + static inline const char* GetMessageStatus(EMessageStatus status) { return ToCString(status); } - + // For lwtrace struct TMessageStatusField { typedef int TStoreType; @@ -53,5 +53,5 @@ namespace NBus { return value; } }; - -} // ns + +} // ns diff --git a/library/cpp/messagebus/message_status_counter.cpp b/library/cpp/messagebus/message_status_counter.cpp index 13d4cfbdab..891c8f5bb2 100644 --- a/library/cpp/messagebus/message_status_counter.cpp +++ b/library/cpp/messagebus/message_status_counter.cpp @@ -1,60 +1,60 @@ #include "message_status_counter.h" -#include "key_value_printer.h" -#include "text_utils.h" - +#include "key_value_printer.h" +#include "text_utils.h" + #include <library/cpp/messagebus/monitoring/mon_proto.pb.h> - + #include <util/stream/str.h> - -using namespace NBus; -using namespace NBus::NPrivate; - + +using namespace NBus; +using namespace NBus::NPrivate; + TMessageStatusCounter::TMessageStatusCounter() { - Zero(Counts); -} - -TMessageStatusCounter& TMessageStatusCounter::operator+=(const TMessageStatusCounter& that) { - for (size_t i = 0; i < MESSAGE_STATUS_COUNT; ++i) { - Counts[i] += that.Counts[i]; - } - return *this; -} - + Zero(Counts); +} + +TMessageStatusCounter& TMessageStatusCounter::operator+=(const TMessageStatusCounter& that) { + for (size_t i = 0; i < MESSAGE_STATUS_COUNT; ++i) { + Counts[i] += that.Counts[i]; + } + return *this; +} + TString TMessageStatusCounter::PrintToString() const { - TStringStream ss; - TKeyValuePrinter p; - bool hasNonZeros = false; - bool hasZeros = false; - for (size_t i = 0; i < MESSAGE_STATUS_COUNT; ++i) { - if (i == MESSAGE_OK) { + TStringStream ss; + TKeyValuePrinter p; + bool hasNonZeros = false; + bool hasZeros = false; + for (size_t i = 0; i < MESSAGE_STATUS_COUNT; ++i) { + if (i == MESSAGE_OK) { Y_VERIFY(Counts[i] == 0); - continue; - } - if (Counts[i] != 0) { - p.AddRow(EMessageStatus(i), Counts[i]); - const char* description = MessageStatusDescription(EMessageStatus(i)); - // TODO: add third column + continue; + } + if (Counts[i] != 0) { + p.AddRow(EMessageStatus(i), Counts[i]); + const char* description = MessageStatusDescription(EMessageStatus(i)); + // TODO: add third column Y_UNUSED(description); - - hasNonZeros = true; - } else { - hasZeros = true; - } - } - if (!hasNonZeros) { - ss << "message status counts are zeros\n"; - } else { - if (hasZeros) { - ss << "message status counts are zeros, except:\n"; - } else { - ss << "message status counts:\n"; - } - ss << IndentText(p.PrintToString()); - } - return ss.Str(); -} - + + hasNonZeros = true; + } else { + hasZeros = true; + } + } + if (!hasNonZeros) { + ss << "message status counts are zeros\n"; + } else { + if (hasZeros) { + ss << "message status counts are zeros, except:\n"; + } else { + ss << "message status counts:\n"; + } + ss << IndentText(p.PrintToString()); + } + return ss.Str(); +} + void TMessageStatusCounter::FillErrorsProtobuf(TConnectionStatusMonRecord* status) const { status->clear_errorcountbystatus(); for (size_t i = 0; i < MESSAGE_STATUS_COUNT; ++i) { diff --git a/library/cpp/messagebus/message_status_counter.h b/library/cpp/messagebus/message_status_counter.h index 18a1afd1b7..e8ba2fdd31 100644 --- a/library/cpp/messagebus/message_status_counter.h +++ b/library/cpp/messagebus/message_status_counter.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include "message_status.h" #include <library/cpp/messagebus/monitoring/mon_proto.pb.h> #include <util/generic/string.h> - + #include <array> - + namespace NBus { namespace NPrivate { struct TMessageStatusCounter { @@ -16,14 +16,14 @@ namespace NBus { } std::array<unsigned, MESSAGE_STATUS_COUNT> Counts; - + unsigned& operator[](EMessageStatus index) { return Counts[index]; } const unsigned& operator[](EMessageStatus index) const { return Counts[index]; } - + TMessageStatusCounter(); TMessageStatusCounter& operator+=(const TMessageStatusCounter&); @@ -31,6 +31,6 @@ namespace NBus { TString PrintToString() const; void FillErrorsProtobuf(TConnectionStatusMonRecord*) const; }; - + } } diff --git a/library/cpp/messagebus/messqueue.cpp b/library/cpp/messagebus/messqueue.cpp index f77e54b548..3474d62705 100644 --- a/library/cpp/messagebus/messqueue.cpp +++ b/library/cpp/messagebus/messqueue.cpp @@ -1,58 +1,58 @@ #include "key_value_printer.h" #include "mb_lwtrace.h" -#include "remote_client_session.h" -#include "remote_server_session.h" +#include "remote_client_session.h" +#include "remote_server_session.h" #include "ybus.h" #include <util/generic/singleton.h> -using namespace NBus; -using namespace NBus::NPrivate; -using namespace NActor; +using namespace NBus; +using namespace NBus::NPrivate; +using namespace NActor; -TBusMessageQueuePtr NBus::CreateMessageQueue(const TBusQueueConfig& config, TExecutorPtr executor, TBusLocator* locator, const char* name) { - return new TBusMessageQueue(config, executor, locator, name); -} - -TBusMessageQueuePtr NBus::CreateMessageQueue(const TBusQueueConfig& config, TBusLocator* locator, const char* name) { - TExecutor::TConfig executorConfig; - executorConfig.WorkerCount = config.NumWorkers; - executorConfig.Name = name; - TExecutorPtr executor = new TExecutor(executorConfig); - return CreateMessageQueue(config, executor, locator, name); +TBusMessageQueuePtr NBus::CreateMessageQueue(const TBusQueueConfig& config, TExecutorPtr executor, TBusLocator* locator, const char* name) { + return new TBusMessageQueue(config, executor, locator, name); } -TBusMessageQueuePtr NBus::CreateMessageQueue(const TBusQueueConfig& config, const char* name) { - return CreateMessageQueue(config, new TBusLocator, name); +TBusMessageQueuePtr NBus::CreateMessageQueue(const TBusQueueConfig& config, TBusLocator* locator, const char* name) { + TExecutor::TConfig executorConfig; + executorConfig.WorkerCount = config.NumWorkers; + executorConfig.Name = name; + TExecutorPtr executor = new TExecutor(executorConfig); + return CreateMessageQueue(config, executor, locator, name); } -TBusMessageQueuePtr NBus::CreateMessageQueue(TExecutorPtr executor, const char* name) { - return CreateMessageQueue(TBusQueueConfig(), executor, new TBusLocator, name); -} - -TBusMessageQueuePtr NBus::CreateMessageQueue(const char* name) { +TBusMessageQueuePtr NBus::CreateMessageQueue(const TBusQueueConfig& config, const char* name) { + return CreateMessageQueue(config, new TBusLocator, name); +} + +TBusMessageQueuePtr NBus::CreateMessageQueue(TExecutorPtr executor, const char* name) { + return CreateMessageQueue(TBusQueueConfig(), executor, new TBusLocator, name); +} + +TBusMessageQueuePtr NBus::CreateMessageQueue(const char* name) { TBusQueueConfig config; - return CreateMessageQueue(config, name); + return CreateMessageQueue(config, name); } -namespace { +namespace { TBusQueueConfig QueueConfigFillDefaults(const TBusQueueConfig& orig, const TString& name) { - TBusQueueConfig patched = orig; - if (!patched.Name) { - patched.Name = name; - } - return patched; - } -} - -TBusMessageQueue::TBusMessageQueue(const TBusQueueConfig& config, TExecutorPtr executor, TBusLocator* locator, const char* name) - : Config(QueueConfigFillDefaults(config, name)) - , Locator(locator) - , WorkQueue(executor) - , Running(1) + TBusQueueConfig patched = orig; + if (!patched.Name) { + patched.Name = name; + } + return patched; + } +} + +TBusMessageQueue::TBusMessageQueue(const TBusQueueConfig& config, TExecutorPtr executor, TBusLocator* locator, const char* name) + : Config(QueueConfigFillDefaults(config, name)) + , Locator(locator) + , WorkQueue(executor) + , Running(1) { - InitBusLwtrace(); - InitNetworkSubSystem(); + InitBusLwtrace(); + InitNetworkSubSystem(); } TBusMessageQueue::~TBusMessageQueue() { @@ -60,73 +60,73 @@ TBusMessageQueue::~TBusMessageQueue() { } void TBusMessageQueue::Stop() { - if (!AtomicCas(&Running, 0, 1)) { - ShutdownComplete.WaitI(); - return; - } - - Scheduler.Stop(); - - DestroyAllSessions(); - - WorkQueue->Stop(); - - ShutdownComplete.Signal(); -} - -bool TBusMessageQueue::IsRunning() { - return AtomicGet(Running); -} - + if (!AtomicCas(&Running, 0, 1)) { + ShutdownComplete.WaitI(); + return; + } + + Scheduler.Stop(); + + DestroyAllSessions(); + + WorkQueue->Stop(); + + ShutdownComplete.Signal(); +} + +bool TBusMessageQueue::IsRunning() { + return AtomicGet(Running); +} + TBusMessageQueueStatus TBusMessageQueue::GetStatusRecordInternal() const { - TBusMessageQueueStatus r; - r.ExecutorStatus = WorkQueue->GetStatusRecordInternal(); - r.Config = Config; - return r; -} - + TBusMessageQueueStatus r; + r.ExecutorStatus = WorkQueue->GetStatusRecordInternal(); + r.Config = Config; + return r; +} + TString TBusMessageQueue::GetStatusSelf() const { - return GetStatusRecordInternal().PrintToString(); -} - + return GetStatusRecordInternal().PrintToString(); +} + TString TBusMessageQueue::GetStatusSingleLine() const { - return WorkQueue->GetStatusSingleLine(); -} - + return WorkQueue->GetStatusSingleLine(); +} + TString TBusMessageQueue::GetStatus(ui16 flags) const { - TStringStream ss; - - ss << GetStatusSelf(); - + TStringStream ss; + + ss << GetStatusSelf(); + TList<TIntrusivePtr<TBusSessionImpl>> sessions; - { - TGuard<TMutex> scope(Lock); - sessions = Sessions; - } - + { + TGuard<TMutex> scope(Lock); + sessions = Sessions; + } + for (TList<TIntrusivePtr<TBusSessionImpl>>::const_iterator session = sessions.begin(); session != sessions.end(); ++session) { - ss << Endl; - ss << (*session)->GetStatus(flags); - } - - ss << Endl; - ss << "object counts (not necessarily owned by this message queue):" << Endl; - TKeyValuePrinter p; - p.AddRow("TRemoteClientConnection", TObjectCounter<TRemoteClientConnection>::ObjectCount(), false); - p.AddRow("TRemoteServerConnection", TObjectCounter<TRemoteServerConnection>::ObjectCount(), false); - p.AddRow("TRemoteClientSession", TObjectCounter<TRemoteClientSession>::ObjectCount(), false); - p.AddRow("TRemoteServerSession", TObjectCounter<TRemoteServerSession>::ObjectCount(), false); - p.AddRow("NEventLoop::TEventLoop", TObjectCounter<NEventLoop::TEventLoop>::ObjectCount(), false); - p.AddRow("NEventLoop::TChannel", TObjectCounter<NEventLoop::TChannel>::ObjectCount(), false); - ss << p.PrintToString(); - - return ss.Str(); -} - + ss << Endl; + ss << (*session)->GetStatus(flags); + } + + ss << Endl; + ss << "object counts (not necessarily owned by this message queue):" << Endl; + TKeyValuePrinter p; + p.AddRow("TRemoteClientConnection", TObjectCounter<TRemoteClientConnection>::ObjectCount(), false); + p.AddRow("TRemoteServerConnection", TObjectCounter<TRemoteServerConnection>::ObjectCount(), false); + p.AddRow("TRemoteClientSession", TObjectCounter<TRemoteClientSession>::ObjectCount(), false); + p.AddRow("TRemoteServerSession", TObjectCounter<TRemoteServerSession>::ObjectCount(), false); + p.AddRow("NEventLoop::TEventLoop", TObjectCounter<NEventLoop::TEventLoop>::ObjectCount(), false); + p.AddRow("NEventLoop::TChannel", TObjectCounter<NEventLoop::TChannel>::ObjectCount(), false); + ss << p.PrintToString(); + + return ss.Str(); +} + TBusClientSessionPtr TBusMessageQueue::CreateSource(TBusProtocol* proto, IBusClientHandler* handler, const TBusClientSessionConfig& config, const TString& name) { - TRemoteClientSessionPtr session(new TRemoteClientSession(this, proto, handler, config, name)); - Add(session.Get()); + TRemoteClientSessionPtr session(new TRemoteClientSession(this, proto, handler, config, name)); + Add(session.Get()); return session.Get(); } @@ -161,27 +161,27 @@ TBusServerSessionPtr TBusMessageQueue::CreateDestination(TBusProtocol* proto, IB } } -void TBusMessageQueue::Add(TIntrusivePtr<TBusSessionImpl> session) { +void TBusMessageQueue::Add(TIntrusivePtr<TBusSessionImpl> session) { TGuard<TMutex> scope(Lock); Sessions.push_back(session); } -void TBusMessageQueue::Remove(TBusSession* session) { - TGuard<TMutex> scope(Lock); +void TBusMessageQueue::Remove(TBusSession* session) { + TGuard<TMutex> scope(Lock); TList<TIntrusivePtr<TBusSessionImpl>>::iterator it = std::find(Sessions.begin(), Sessions.end(), session); Y_VERIFY(it != Sessions.end(), "do not destroy session twice"); - Sessions.erase(it); -} - + Sessions.erase(it); +} + void TBusMessageQueue::Destroy(TBusSession* session) { - session->Shutdown(); -} - -void TBusMessageQueue::DestroyAllSessions() { + session->Shutdown(); +} + +void TBusMessageQueue::DestroyAllSessions() { TList<TIntrusivePtr<TBusSessionImpl>> sessions; { TGuard<TMutex> scope(Lock); - sessions = Sessions; + sessions = Sessions; } for (auto& session : sessions) { @@ -194,5 +194,5 @@ void TBusMessageQueue::Schedule(IScheduleItemAutoPtr i) { } TString TBusMessageQueue::GetNameInternal() const { - return Config.Name; -} + return Config.Name; +} diff --git a/library/cpp/messagebus/misc/atomic_box.h b/library/cpp/messagebus/misc/atomic_box.h index a7e83b70ab..401621f933 100644 --- a/library/cpp/messagebus/misc/atomic_box.h +++ b/library/cpp/messagebus/misc/atomic_box.h @@ -1,34 +1,34 @@ -#pragma once - +#pragma once + #include <util/system/atomic.h> -// TAtomic with human interface -template <typename T> -class TAtomicBox { -private: - union { - TAtomic Value; - // when T is enum, it is convenient to inspect its content in gdb - T ValueForDebugger; - }; - +// TAtomic with human interface +template <typename T> +class TAtomicBox { +private: + union { + TAtomic Value; + // when T is enum, it is convenient to inspect its content in gdb + T ValueForDebugger; + }; + static_assert(sizeof(T) <= sizeof(TAtomic), "expect sizeof(T) <= sizeof(TAtomic)"); -public: +public: TAtomicBox(T value = T()) : Value(value) { } - - void Set(T value) { + + void Set(T value) { AtomicSet(Value, (TAtomic)value); - } - - T Get() const { + } + + T Get() const { return (T)AtomicGet(Value); - } - - bool CompareAndSet(T expected, T set) { + } + + bool CompareAndSet(T expected, T set) { return AtomicCas(&Value, (TAtomicBase)set, (TAtomicBase)expected); - } -}; + } +}; diff --git a/library/cpp/messagebus/misc/test_sync.h b/library/cpp/messagebus/misc/test_sync.h index 8e5aa212aa..be3f4f20b8 100644 --- a/library/cpp/messagebus/misc/test_sync.h +++ b/library/cpp/messagebus/misc/test_sync.h @@ -1,75 +1,75 @@ -#pragma once - +#pragma once + #include <util/system/condvar.h> -#include <util/system/mutex.h> - -class TTestSync { -private: - unsigned Current; - - TMutex Mutex; - TCondVar CondVar; - -public: - TTestSync() - : Current(0) +#include <util/system/mutex.h> + +class TTestSync { +private: + unsigned Current; + + TMutex Mutex; + TCondVar CondVar; + +public: + TTestSync() + : Current(0) { } - - void Inc() { - TGuard<TMutex> guard(Mutex); - - DoInc(); - CondVar.BroadCast(); - } - - unsigned Get() { - TGuard<TMutex> guard(Mutex); - - return Current; - } - - void WaitFor(unsigned n) { - TGuard<TMutex> guard(Mutex); - + + void Inc() { + TGuard<TMutex> guard(Mutex); + + DoInc(); + CondVar.BroadCast(); + } + + unsigned Get() { + TGuard<TMutex> guard(Mutex); + + return Current; + } + + void WaitFor(unsigned n) { + TGuard<TMutex> guard(Mutex); + Y_VERIFY(Current <= n, "too late, waiting for %d, already %d", n, Current); - - while (n > Current) { - CondVar.WaitI(Mutex); - } - } - - void WaitForAndIncrement(unsigned n) { - TGuard<TMutex> guard(Mutex); - + + while (n > Current) { + CondVar.WaitI(Mutex); + } + } + + void WaitForAndIncrement(unsigned n) { + TGuard<TMutex> guard(Mutex); + Y_VERIFY(Current <= n, "too late, waiting for %d, already %d", n, Current); - - while (n > Current) { - CondVar.WaitI(Mutex); - } - - DoInc(); - CondVar.BroadCast(); - } - - void CheckAndIncrement(unsigned n) { - TGuard<TMutex> guard(Mutex); - + + while (n > Current) { + CondVar.WaitI(Mutex); + } + + DoInc(); + CondVar.BroadCast(); + } + + void CheckAndIncrement(unsigned n) { + TGuard<TMutex> guard(Mutex); + Y_VERIFY(Current == n, "must be %d, currently %d", n, Current); - - DoInc(); - CondVar.BroadCast(); - } - - void Check(unsigned n) { - TGuard<TMutex> guard(Mutex); - + + DoInc(); + CondVar.BroadCast(); + } + + void Check(unsigned n) { + TGuard<TMutex> guard(Mutex); + Y_VERIFY(Current == n, "must be %d, currently %d", n, Current); - } - -private: - void DoInc() { - unsigned r = ++Current; + } + +private: + void DoInc() { + unsigned r = ++Current; Y_UNUSED(r); - } -}; + } +}; diff --git a/library/cpp/messagebus/misc/weak_ptr.h b/library/cpp/messagebus/misc/weak_ptr.h index 46b6496d86..70fdeb0e2a 100644 --- a/library/cpp/messagebus/misc/weak_ptr.h +++ b/library/cpp/messagebus/misc/weak_ptr.h @@ -1,99 +1,99 @@ -#pragma once - -#include <util/generic/ptr.h> -#include <util/system/mutex.h> - -template <typename T> -struct TWeakPtr; - -template <typename TSelf> -struct TWeakRefCounted { +#pragma once + +#include <util/generic/ptr.h> +#include <util/system/mutex.h> + +template <typename T> +struct TWeakPtr; + +template <typename TSelf> +struct TWeakRefCounted { template <typename> friend struct TWeakPtr; -private: +private: struct TRef: public TAtomicRefCount<TRef> { - TMutex Mutex; - TSelf* Outer; - + TMutex Mutex; + TSelf* Outer; + TRef(TSelf* outer) : Outer(outer) { } - - void Release() { - TGuard<TMutex> g(Mutex); + + void Release() { + TGuard<TMutex> g(Mutex); Y_ASSERT(!!Outer); Outer = nullptr; - } - - TIntrusivePtr<TSelf> Get() { - TGuard<TMutex> g(Mutex); + } + + TIntrusivePtr<TSelf> Get() { + TGuard<TMutex> g(Mutex); Y_ASSERT(!Outer || Outer->RefCount() > 0); - return Outer; - } - }; - - TAtomicCounter Counter; - TIntrusivePtr<TRef> RefPtr; - -public: - TWeakRefCounted() - : RefPtr(new TRef(static_cast<TSelf*>(this))) + return Outer; + } + }; + + TAtomicCounter Counter; + TIntrusivePtr<TRef> RefPtr; + +public: + TWeakRefCounted() + : RefPtr(new TRef(static_cast<TSelf*>(this))) { } - - void Ref() { - Counter.Inc(); - } - - void UnRef() { - if (Counter.Dec() == 0) { - RefPtr->Release(); - - // drop is to prevent dtor from reading it - RefPtr.Drop(); - - delete static_cast<TSelf*>(this); - } - } - - void DecRef() { - Counter.Dec(); - } - - unsigned RefCount() const { - return Counter.Val(); - } -}; - -template <typename T> -struct TWeakPtr { -private: - typedef TIntrusivePtr<typename T::TRef> TRefPtr; - TRefPtr RefPtr; - -public: + + void Ref() { + Counter.Inc(); + } + + void UnRef() { + if (Counter.Dec() == 0) { + RefPtr->Release(); + + // drop is to prevent dtor from reading it + RefPtr.Drop(); + + delete static_cast<TSelf*>(this); + } + } + + void DecRef() { + Counter.Dec(); + } + + unsigned RefCount() const { + return Counter.Val(); + } +}; + +template <typename T> +struct TWeakPtr { +private: + typedef TIntrusivePtr<typename T::TRef> TRefPtr; + TRefPtr RefPtr; + +public: TWeakPtr() { } - - TWeakPtr(T* t) { - if (!!t) { - RefPtr = t->RefPtr; - } - } - - TWeakPtr(TIntrusivePtr<T> t) { - if (!!t) { - RefPtr = t->RefPtr; - } - } - - TIntrusivePtr<T> Get() { - if (!RefPtr) { + + TWeakPtr(T* t) { + if (!!t) { + RefPtr = t->RefPtr; + } + } + + TWeakPtr(TIntrusivePtr<T> t) { + if (!!t) { + RefPtr = t->RefPtr; + } + } + + TIntrusivePtr<T> Get() { + if (!RefPtr) { return nullptr; - } else { - return RefPtr->Get(); - } - } -}; + } else { + return RefPtr->Get(); + } + } +}; diff --git a/library/cpp/messagebus/misc/weak_ptr_ut.cpp b/library/cpp/messagebus/misc/weak_ptr_ut.cpp index 63d253e128..5a325278db 100644 --- a/library/cpp/messagebus/misc/weak_ptr_ut.cpp +++ b/library/cpp/messagebus/misc/weak_ptr_ut.cpp @@ -1,11 +1,11 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "weak_ptr.h" - + +#include "weak_ptr.h" + Y_UNIT_TEST_SUITE(TWeakPtrTest) { - struct TWeakPtrTester: public TWeakRefCounted<TWeakPtrTester> { - int* const CounterPtr; - + struct TWeakPtrTester: public TWeakRefCounted<TWeakPtrTester> { + int* const CounterPtr; + TWeakPtrTester(int* counterPtr) : CounterPtr(counterPtr) { @@ -13,34 +13,34 @@ Y_UNIT_TEST_SUITE(TWeakPtrTest) { ~TWeakPtrTester() { ++*CounterPtr; } - }; - + }; + Y_UNIT_TEST(Simple) { - int destroyCount = 0; - - TIntrusivePtr<TWeakPtrTester> p(new TWeakPtrTester(&destroyCount)); - - UNIT_ASSERT(!!p); - UNIT_ASSERT_VALUES_EQUAL(1u, p->RefCount()); - - TWeakPtr<TWeakPtrTester> p2(p); - - UNIT_ASSERT_VALUES_EQUAL(1u, p->RefCount()); - - { - TIntrusivePtr<TWeakPtrTester> p3 = p2.Get(); - UNIT_ASSERT(!!p3); - UNIT_ASSERT_VALUES_EQUAL(2u, p->RefCount()); - } - - p.Drop(); - UNIT_ASSERT_VALUES_EQUAL(1, destroyCount); - - { - TIntrusivePtr<TWeakPtrTester> p3 = p2.Get(); - UNIT_ASSERT(!p3); - } - - UNIT_ASSERT_VALUES_EQUAL(1, destroyCount); - } -} + int destroyCount = 0; + + TIntrusivePtr<TWeakPtrTester> p(new TWeakPtrTester(&destroyCount)); + + UNIT_ASSERT(!!p); + UNIT_ASSERT_VALUES_EQUAL(1u, p->RefCount()); + + TWeakPtr<TWeakPtrTester> p2(p); + + UNIT_ASSERT_VALUES_EQUAL(1u, p->RefCount()); + + { + TIntrusivePtr<TWeakPtrTester> p3 = p2.Get(); + UNIT_ASSERT(!!p3); + UNIT_ASSERT_VALUES_EQUAL(2u, p->RefCount()); + } + + p.Drop(); + UNIT_ASSERT_VALUES_EQUAL(1, destroyCount); + + { + TIntrusivePtr<TWeakPtrTester> p3 = p2.Get(); + UNIT_ASSERT(!p3); + } + + UNIT_ASSERT_VALUES_EQUAL(1, destroyCount); + } +} diff --git a/library/cpp/messagebus/monitoring/mon_proto.proto b/library/cpp/messagebus/monitoring/mon_proto.proto index cfc1cebe26..73b6614481 100644 --- a/library/cpp/messagebus/monitoring/mon_proto.proto +++ b/library/cpp/messagebus/monitoring/mon_proto.proto @@ -1,6 +1,6 @@ import "library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto"; - -package NBus; + +package NBus; option java_package = "ru.yandex.messagebus.monitoring.proto"; @@ -28,28 +28,28 @@ message TMessageStatusRecord { optional uint32 Count = 2; } -message TConnectionStatusMonRecord { +message TConnectionStatusMonRecord { optional uint32 SendQueueSize = 1 [ (NMonProto.Metric).Type = GAUGE ]; - // client only + // client only optional uint32 AckMessagesSize = 2 [ (NMonProto.Metric).Type = GAUGE ]; optional uint32 ErrorCount = 3 [ (NMonProto.Metric).Type = RATE ]; - + optional uint64 WriteBytes = 10 [ (NMonProto.Metric).Type = RATE ]; - optional uint64 WriteBytesCompressed = 11; + optional uint64 WriteBytesCompressed = 11; optional uint64 WriteMessages = 12 [ (NMonProto.Metric).Type = RATE ]; - optional uint64 WriteSyscalls = 13; - optional uint64 WriteActs = 14; + optional uint64 WriteSyscalls = 13; + optional uint64 WriteActs = 14; optional uint64 ReadBytes = 20 [ (NMonProto.Metric).Type = RATE ]; - optional uint64 ReadBytesCompressed = 21; + optional uint64 ReadBytesCompressed = 21; optional uint64 ReadMessages = 22 [ (NMonProto.Metric).Type = RATE ]; - optional uint64 ReadSyscalls = 23; - optional uint64 ReadActs = 24; + optional uint64 ReadSyscalls = 23; + optional uint64 ReadActs = 24; repeated TMessageStatusRecord ErrorCountByStatus = 25; -} - -message TSessionStatusMonRecord { +} + +message TSessionStatusMonRecord { optional uint32 InFlight = 1 [ (NMonProto.Metric).Type = GAUGE ]; optional uint32 ConnectionCount = 2 [ (NMonProto.Metric).Type = GAUGE ]; optional uint32 ConnectCount = 3 [ (NMonProto.Metric).Type = RATE ]; -} +} diff --git a/library/cpp/messagebus/monitoring/ya.make b/library/cpp/messagebus/monitoring/ya.make index 07df343142..25782492b1 100644 --- a/library/cpp/messagebus/monitoring/ya.make +++ b/library/cpp/messagebus/monitoring/ya.make @@ -1,15 +1,15 @@ PROTO_LIBRARY() - + OWNER(g:messagebus) - -PEERDIR( + +PEERDIR( library/cpp/monlib/encode/legacy_protobuf/protos -) - -SRCS( - mon_proto.proto -) - +) + +SRCS( + mon_proto.proto +) + EXCLUDE_TAGS(GO_PROTO) -END() +END() diff --git a/library/cpp/messagebus/moved.h b/library/cpp/messagebus/moved.h index fe3fd476b4..ede8dcd244 100644 --- a/library/cpp/messagebus/moved.h +++ b/library/cpp/messagebus/moved.h @@ -1,39 +1,39 @@ -#pragma once - -#include <util/generic/utility.h> - -template <typename T> -class TMoved { -private: - mutable T Value; - -public: +#pragma once + +#include <util/generic/utility.h> + +template <typename T> +class TMoved { +private: + mutable T Value; + +public: TMoved() { } - TMoved(const TMoved<T>& that) { - DoSwap(Value, that.Value); - } - TMoved(const T& that) { - DoSwap(Value, const_cast<T&>(that)); - } - - void swap(TMoved& that) { - DoSwap(Value, that.Value); - } - - T& operator*() { - return Value; - } - - const T& operator*() const { - return Value; - } - - T* operator->() { - return &Value; - } - - const T* operator->() const { - return &Value; - } -}; + TMoved(const TMoved<T>& that) { + DoSwap(Value, that.Value); + } + TMoved(const T& that) { + DoSwap(Value, const_cast<T&>(that)); + } + + void swap(TMoved& that) { + DoSwap(Value, that.Value); + } + + T& operator*() { + return Value; + } + + const T& operator*() const { + return Value; + } + + T* operator->() { + return &Value; + } + + const T* operator->() const { + return &Value; + } +}; diff --git a/library/cpp/messagebus/moved_ut.cpp b/library/cpp/messagebus/moved_ut.cpp index bdae344db6..c1a07cce7e 100644 --- a/library/cpp/messagebus/moved_ut.cpp +++ b/library/cpp/messagebus/moved_ut.cpp @@ -1,22 +1,22 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "moved.h" - + +#include "moved.h" + Y_UNIT_TEST_SUITE(TMovedTest) { Y_UNIT_TEST(Simple) { TMoved<THolder<int>> h1(MakeHolder<int>(10)); TMoved<THolder<int>> h2 = h1; - UNIT_ASSERT(!*h1); - UNIT_ASSERT(!!*h2); - UNIT_ASSERT_VALUES_EQUAL(10, **h2); - } - + UNIT_ASSERT(!*h1); + UNIT_ASSERT(!!*h2); + UNIT_ASSERT_VALUES_EQUAL(10, **h2); + } + void Foo(TMoved<THolder<int>> h) { - UNIT_ASSERT_VALUES_EQUAL(11, **h); - } - + UNIT_ASSERT_VALUES_EQUAL(11, **h); + } + Y_UNIT_TEST(PassToFunction) { - THolder<int> h(new int(11)); - Foo(h); - } -} + THolder<int> h(new int(11)); + Foo(h); + } +} diff --git a/library/cpp/messagebus/netaddr_ut.cpp b/library/cpp/messagebus/netaddr_ut.cpp index e9b7aafc1d..e5c68bf402 100644 --- a/library/cpp/messagebus/netaddr_ut.cpp +++ b/library/cpp/messagebus/netaddr_ut.cpp @@ -1,21 +1,21 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "netaddr.h" + +#include "netaddr.h" #include "test_utils.h" - -using namespace NBus; - + +using namespace NBus; + Y_UNIT_TEST_SUITE(TNetAddr) { Y_UNIT_TEST(ResolveIpv4) { ASSUME_IP_V4_ENABLED; - UNIT_ASSERT(TNetAddr("ns1.yandex.ru", 80, EIP_VERSION_4).IsIpv4()); - } - + UNIT_ASSERT(TNetAddr("ns1.yandex.ru", 80, EIP_VERSION_4).IsIpv4()); + } + Y_UNIT_TEST(ResolveIpv6) { - UNIT_ASSERT(TNetAddr("ns1.yandex.ru", 80, EIP_VERSION_6).IsIpv6()); - } - + UNIT_ASSERT(TNetAddr("ns1.yandex.ru", 80, EIP_VERSION_6).IsIpv6()); + } + Y_UNIT_TEST(ResolveAny) { - TNetAddr("ns1.yandex.ru", 80, EIP_VERSION_ANY); - } -} + TNetAddr("ns1.yandex.ru", 80, EIP_VERSION_ANY); + } +} diff --git a/library/cpp/messagebus/network.cpp b/library/cpp/messagebus/network.cpp index 5a0ab5163d..304bedae5a 100644 --- a/library/cpp/messagebus/network.cpp +++ b/library/cpp/messagebus/network.cpp @@ -3,27 +3,27 @@ #include <util/generic/maybe.h> #include <util/generic/ptr.h> #include <util/network/init.h> -#include <util/network/socket.h> +#include <util/network/socket.h> #include <util/system/platform.h> - -using namespace NBus; -using namespace NBus::NPrivate; - -namespace { + +using namespace NBus; +using namespace NBus::NPrivate; + +namespace { TBindResult BindOnPortProto(int port, int af, bool reusePort) { Y_VERIFY(af == AF_INET || af == AF_INET6, "wrong af"); - - SOCKET fd = ::socket(af, SOCK_STREAM, 0); + + SOCKET fd = ::socket(af, SOCK_STREAM, 0); if (fd == INVALID_SOCKET) { ythrow TSystemError() << "failed to create a socket"; } - - int one = 1; + + int one = 1; int r1 = SetSockOpt(fd, SOL_SOCKET, SO_REUSEADDR, one); if (r1 < 0) { ythrow TSystemError() << "failed to setsockopt SO_REUSEADDR"; } - + #ifdef SO_REUSEPORT if (reusePort) { int r = SetSockOpt(fd, SOL_SOCKET, SO_REUSEPORT, one); @@ -35,43 +35,43 @@ namespace { Y_UNUSED(reusePort); #endif - THolder<TOpaqueAddr> addr(new TOpaqueAddr); - sockaddr* sa = addr->MutableAddr(); - sa->sa_family = af; - socklen_t len; - if (af == AF_INET) { - len = sizeof(sockaddr_in); + THolder<TOpaqueAddr> addr(new TOpaqueAddr); + sockaddr* sa = addr->MutableAddr(); + sa->sa_family = af; + socklen_t len; + if (af == AF_INET) { + len = sizeof(sockaddr_in); ((sockaddr_in*)sa)->sin_port = HostToInet((ui16)port); ((sockaddr_in*)sa)->sin_addr.s_addr = INADDR_ANY; - } else { - len = sizeof(sockaddr_in6); + } else { + len = sizeof(sockaddr_in6); ((sockaddr_in6*)sa)->sin6_port = HostToInet((ui16)port); - } - - if (af == AF_INET6) { - FixIPv6ListenSocket(fd); - } - - int r2 = ::bind(fd, sa, len); - if (r2 < 0) { + } + + if (af == AF_INET6) { + FixIPv6ListenSocket(fd); + } + + int r2 = ::bind(fd, sa, len); + if (r2 < 0) { ythrow TSystemError() << "failed to bind on port " << port; - } - - int rsn = ::getsockname(fd, addr->MutableAddr(), addr->LenPtr()); + } + + int rsn = ::getsockname(fd, addr->MutableAddr(), addr->LenPtr()); if (rsn < 0) { ythrow TSystemError() << "failed to getsockname"; } - - int r3 = ::listen(fd, 50); + + int r3 = ::listen(fd, 50); if (r3 < 0) { ythrow TSystemError() << "listen failed"; } - - TBindResult r; - r.Socket.Reset(new TSocketHolder(fd)); - r.Addr = TNetAddr(addr.Release()); - return r; - } + + TBindResult r; + r.Socket.Reset(new TSocketHolder(fd)); + r.Addr = TNetAddr(addr.Release()); + return r; + } TMaybe<TBindResult> TryBindOnPortProto(int port, int af, bool reusePort) { try { @@ -91,13 +91,13 @@ namespace { r.second.emplace_back(std::move(r2)); return r; } -} - +} + std::pair<unsigned, TVector<TBindResult>> NBus::BindOnPort(int port, bool reusePort) { std::pair<unsigned, TVector<TBindResult>> r; r.second.reserve(2); - - if (port != 0) { + + if (port != 0) { return AggregateBindResults(BindOnPortProto(port, AF_INET, reusePort), BindOnPortProto(port, AF_INET6, reusePort)); } @@ -107,50 +107,50 @@ std::pair<unsigned, TVector<TBindResult>> NBus::BindOnPort(int port, bool reuseP TMaybe<TBindResult> in4 = TryBindOnPortProto(0, AF_INET, reusePort); if (!in4) { continue; - } + } TMaybe<TBindResult> in6 = TryBindOnPortProto(in4->Addr.GetPort(), AF_INET6, reusePort); if (!in6) { continue; - } + } return AggregateBindResults(std::move(*in4), std::move(*in6)); - } + } TBindResult in4 = BindOnPortProto(0, AF_INET, reusePort); TBindResult in6 = BindOnPortProto(in4.Addr.GetPort(), AF_INET6, reusePort); return AggregateBindResults(std::move(in4), std::move(in6)); -} - -void NBus::NPrivate::SetSockOptTcpCork(SOCKET s, bool value) { -#ifdef _linux_ +} + +void NBus::NPrivate::SetSockOptTcpCork(SOCKET s, bool value) { +#ifdef _linux_ CheckedSetSockOpt(s, IPPROTO_TCP, TCP_CORK, (int)value, "TCP_CORK"); #else Y_UNUSED(s); Y_UNUSED(value); -#endif -} - +#endif +} + ssize_t NBus::NPrivate::SocketSend(SOCKET s, TArrayRef<const char> data) { - int flags = 0; -#if defined(_linux_) || defined(_freebsd_) - flags |= MSG_NOSIGNAL; -#endif - ssize_t r = ::send(s, data.data(), data.size(), flags); - if (r < 0) { + int flags = 0; +#if defined(_linux_) || defined(_freebsd_) + flags |= MSG_NOSIGNAL; +#endif + ssize_t r = ::send(s, data.data(), data.size(), flags); + if (r < 0) { Y_VERIFY(LastSystemError() != EBADF, "bad fd"); - } - return r; -} - + } + return r; +} + ssize_t NBus::NPrivate::SocketRecv(SOCKET s, TArrayRef<char> buffer) { - int flags = 0; -#if defined(_linux_) || defined(_freebsd_) - flags |= MSG_NOSIGNAL; -#endif - ssize_t r = ::recv(s, buffer.data(), buffer.size(), flags); - if (r < 0) { + int flags = 0; +#if defined(_linux_) || defined(_freebsd_) + flags |= MSG_NOSIGNAL; +#endif + ssize_t r = ::recv(s, buffer.data(), buffer.size(), flags); + if (r < 0) { Y_VERIFY(LastSystemError() != EBADF, "bad fd"); - } - return r; -} + } + return r; +} diff --git a/library/cpp/messagebus/network.h b/library/cpp/messagebus/network.h index 2bb347c38b..cc4bd76ea3 100644 --- a/library/cpp/messagebus/network.h +++ b/library/cpp/messagebus/network.h @@ -1,28 +1,28 @@ -#pragma once - +#pragma once + #include "netaddr.h" - + #include <util/generic/array_ref.h> #include <util/generic/ptr.h> -#include <util/network/socket.h> - +#include <util/network/socket.h> + #include <utility> - + namespace NBus { namespace NPrivate { void SetSockOptTcpCork(SOCKET s, bool value); - + [[nodiscard]] ssize_t SocketSend(SOCKET s, TArrayRef<const char> data); - + [[nodiscard]] ssize_t SocketRecv(SOCKET s, TArrayRef<char> buffer); - + } - + struct TBindResult { TSimpleSharedPtr<TSocketHolder> Socket; TNetAddr Addr; }; - + std::pair<unsigned, TVector<TBindResult>> BindOnPort(int port, bool reusePort); } diff --git a/library/cpp/messagebus/network_ut.cpp b/library/cpp/messagebus/network_ut.cpp index 4091b8ed7f..f1798419db 100644 --- a/library/cpp/messagebus/network_ut.cpp +++ b/library/cpp/messagebus/network_ut.cpp @@ -1,65 +1,65 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "network.h" - + +#include "network.h" + #include <library/cpp/messagebus/test/helper/fixed_port.h> -using namespace NBus; -using namespace NBus::NPrivate; -using namespace NBus::NTest; - -namespace { - int GetSockPort(SOCKET socket) { - sockaddr_storage addr; - Zero(addr); - - socklen_t len = sizeof(addr); - +using namespace NBus; +using namespace NBus::NPrivate; +using namespace NBus::NTest; + +namespace { + int GetSockPort(SOCKET socket) { + sockaddr_storage addr; + Zero(addr); + + socklen_t len = sizeof(addr); + int r = ::getsockname(socket, (sockaddr*)&addr, &len); - UNIT_ASSERT(r >= 0); - - if (addr.ss_family == AF_INET) { + UNIT_ASSERT(r >= 0); + + if (addr.ss_family == AF_INET) { sockaddr_in* addr_in = (sockaddr_in*)&addr; - return InetToHost(addr_in->sin_port); - } else if (addr.ss_family == AF_INET6) { + return InetToHost(addr_in->sin_port); + } else if (addr.ss_family == AF_INET6) { sockaddr_in6* addr_in6 = (sockaddr_in6*)&addr; - return InetToHost(addr_in6->sin6_port); - } else { - UNIT_FAIL("unknown AF"); - throw 1; - } - } -} - + return InetToHost(addr_in6->sin6_port); + } else { + UNIT_FAIL("unknown AF"); + throw 1; + } + } +} + Y_UNIT_TEST_SUITE(Network) { Y_UNIT_TEST(BindOnPortConcrete) { - if (!IsFixedPortTestAllowed()) { - return; - } - + if (!IsFixedPortTestAllowed()) { + return; + } + TVector<TBindResult> r = BindOnPort(FixedPort, false).second; - UNIT_ASSERT_VALUES_EQUAL(size_t(2), r.size()); - + UNIT_ASSERT_VALUES_EQUAL(size_t(2), r.size()); + for (TVector<TBindResult>::iterator i = r.begin(); i != r.end(); ++i) { - UNIT_ASSERT_VALUES_EQUAL(i->Addr.GetPort(), GetSockPort(i->Socket->operator SOCKET())); - } - } - + UNIT_ASSERT_VALUES_EQUAL(i->Addr.GetPort(), GetSockPort(i->Socket->operator SOCKET())); + } + } + Y_UNIT_TEST(BindOnPortRandom) { TVector<TBindResult> r = BindOnPort(0, false).second; - UNIT_ASSERT_VALUES_EQUAL(size_t(2), r.size()); - + UNIT_ASSERT_VALUES_EQUAL(size_t(2), r.size()); + for (TVector<TBindResult>::iterator i = r.begin(); i != r.end(); ++i) { - UNIT_ASSERT_VALUES_EQUAL(i->Addr.GetPort(), GetSockPort(i->Socket->operator SOCKET())); - UNIT_ASSERT(i->Addr.GetPort() > 0); - } - - UNIT_ASSERT_VALUES_EQUAL(r.at(0).Addr.GetPort(), r.at(1).Addr.GetPort()); - } + UNIT_ASSERT_VALUES_EQUAL(i->Addr.GetPort(), GetSockPort(i->Socket->operator SOCKET())); + UNIT_ASSERT(i->Addr.GetPort() > 0); + } + + UNIT_ASSERT_VALUES_EQUAL(r.at(0).Addr.GetPort(), r.at(1).Addr.GetPort()); + } Y_UNIT_TEST(BindOnBusyPort) { auto r = BindOnPort(0, false); UNIT_ASSERT_EXCEPTION_CONTAINS(BindOnPort(r.first, false), TSystemError, "failed to bind on port " + ToString(r.first)); } -} +} diff --git a/library/cpp/messagebus/nondestroying_holder.h b/library/cpp/messagebus/nondestroying_holder.h index 7349ed1381..f4725d696f 100644 --- a/library/cpp/messagebus/nondestroying_holder.h +++ b/library/cpp/messagebus/nondestroying_holder.h @@ -1,39 +1,39 @@ -#pragma once - -#include <util/generic/ptr.h> - -template <typename T> -class TNonDestroyingHolder: public THolder<T> { -public: +#pragma once + +#include <util/generic/ptr.h> + +template <typename T> +class TNonDestroyingHolder: public THolder<T> { +public: TNonDestroyingHolder(T* t = nullptr) noexcept - : THolder<T>(t) - { - } - + : THolder<T>(t) + { + } + TNonDestroyingHolder(TAutoPtr<T> t) noexcept - : THolder<T>(t) - { - } - - ~TNonDestroyingHolder() { + : THolder<T>(t) + { + } + + ~TNonDestroyingHolder() { Y_VERIFY(!*this, "stored object must be explicitly released"); - } -}; - -template <class T> -class TNonDestroyingAutoPtr: public TAutoPtr<T> { -public: + } +}; + +template <class T> +class TNonDestroyingAutoPtr: public TAutoPtr<T> { +public: inline TNonDestroyingAutoPtr(T* t = 0) noexcept - : TAutoPtr<T>(t) - { - } - + : TAutoPtr<T>(t) + { + } + inline TNonDestroyingAutoPtr(const TAutoPtr<T>& t) noexcept - : TAutoPtr<T>(t.Release()) - { - } - + : TAutoPtr<T>(t.Release()) + { + } + inline ~TNonDestroyingAutoPtr() { Y_VERIFY(!*this, "stored object must be explicitly released"); - } -}; + } +}; diff --git a/library/cpp/messagebus/nondestroying_holder_ut.cpp b/library/cpp/messagebus/nondestroying_holder_ut.cpp index 890dbc7b11..208042a2ba 100644 --- a/library/cpp/messagebus/nondestroying_holder_ut.cpp +++ b/library/cpp/messagebus/nondestroying_holder_ut.cpp @@ -1,12 +1,12 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "nondestroying_holder.h" - + +#include "nondestroying_holder.h" + Y_UNIT_TEST_SUITE(TNonDestroyingHolder) { Y_UNIT_TEST(ToAutoPtr) { - TNonDestroyingHolder<int> h(new int(11)); - TAutoPtr<int> i(h); - UNIT_ASSERT_VALUES_EQUAL(11, *i); - UNIT_ASSERT(!h); - } -} + TNonDestroyingHolder<int> h(new int(11)); + TAutoPtr<int> i(h); + UNIT_ASSERT_VALUES_EQUAL(11, *i); + UNIT_ASSERT(!h); + } +} diff --git a/library/cpp/messagebus/oldmodule/module.cpp b/library/cpp/messagebus/oldmodule/module.cpp index ccebcfb7cc..24bd778799 100644 --- a/library/cpp/messagebus/oldmodule/module.cpp +++ b/library/cpp/messagebus/oldmodule/module.cpp @@ -1,4 +1,4 @@ -#include "module.h" +#include "module.h" #include <library/cpp/messagebus/scheduler_actor.h> #include <library/cpp/messagebus/thread_extra.h> @@ -9,19 +9,19 @@ #include <util/generic/singleton.h> #include <util/string/printf.h> -#include <util/system/event.h> +#include <util/system/event.h> + +using namespace NActor; +using namespace NBus; +using namespace NBus::NPrivate; -using namespace NActor; -using namespace NBus; -using namespace NBus::NPrivate; - namespace { Y_POD_STATIC_THREAD(TBusJob*) ThreadCurrentJob; struct TThreadCurrentJobGuard { TBusJob* Prev; - + TThreadCurrentJobGuard(TBusJob* job) : Prev(ThreadCurrentJob) { @@ -32,7 +32,7 @@ namespace { ThreadCurrentJob = Prev; } }; - + void ClearState(NBus::TJobState* state) { /// skip sendbacks handlers if (state->Message != state->Reply) { @@ -46,11 +46,11 @@ namespace { state->Reply = nullptr; } } - } - + } + void ClearJobStateVector(NBus::TJobStateVec* vec) { Y_ASSERT(vec); - + for (auto& call : *vec) { ClearState(&call); } @@ -71,12 +71,12 @@ namespace NBus { : Module(module) { } - + void OnReply(TAutoPtr<TBusMessage> req, TAutoPtr<TBusMessage> reply) override; void OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage) override; void OnError(TAutoPtr<TBusMessage> msg, EMessageStatus status) override; void OnClientConnectionEvent(const TClientConnectionEvent& event) override; - + TBusModuleImpl* const Module; }; @@ -94,13 +94,13 @@ namespace NBus { struct TBusModuleImpl: public TBusModuleInternal { TBusModule* const Module; - + TBusMessageQueue* Queue; - + TScheduler Scheduler; - + const char* const Name; - + typedef TList<TJobRunner*> TBusJobList; /// jobs currently in-flight on this module TBusJobList Jobs; @@ -108,13 +108,13 @@ namespace NBus { TMutex Lock; TCondVar ShutdownCondVar; TAtomic JobCount; - + enum EState { CREATED, RUNNING, STOPPED, }; - + TAtomic State; TBusModuleConfig ModuleConfig; TBusServerSessionPtr ExternalSession; @@ -122,12 +122,12 @@ namespace NBus { THolder<IBusClientHandler> ModuleClientHandler; THolder<IBusServerHandler> ModuleServerHandler; TVector<TSimpleSharedPtr<TBusStarter>> Starters; - + // Sessions must be destroyed before // ModuleClientHandler / ModuleServerHandler TVector<TBusClientSessionPtr> ClientSessions; TVector<TBusServerSessionPtr> ServerSessions; - + TBusModuleImpl(TBusModule* module, const char* name) : Module(module) , Queue() @@ -139,12 +139,12 @@ namespace NBus { , ModuleServerHandler(new TModuleServerHandler(this)) { } - + ~TBusModuleImpl() override { // Shutdown cannot be called from destructor, // because module has virtual methods. Y_VERIFY(State != RUNNING, "if running, must explicitly call Shutdown() before destructor"); - + Scheduler.Stop(); while (!Jobs.empty()) { @@ -152,56 +152,56 @@ namespace NBus { } Y_VERIFY(JobCount == 0, "state check"); } - + void OnMessageReceived(TAutoPtr<TBusMessage> msg, TOnMessageContext&); - + void AddJob(TJobRunner* jobRunner); - + void DestroyJob(TJobRunner* job); - + /// terminate job on this message void CancelJob(TBusJob* job, EMessageStatus status); /// prints statuses of jobs TString GetStatus(unsigned flags); - + size_t Size() const { return AtomicGet(JobCount); } - + void Shutdown(); - + TVector<TBusClientSessionPtr> GetClientSessionsInternal() override { return ClientSessions; } - + TVector<TBusServerSessionPtr> GetServerSessionsInternal() override { return ServerSessions; } - + TBusMessageQueue* GetQueue() override { return Queue; } - + TString GetNameInternal() override { return Name; } - + TString GetStatusSingleLine() override { TStringStream ss; ss << "jobs: " << Size(); return ss.Str(); } - + void OnClientConnectionEvent(const TClientConnectionEvent& event) { Module->OnClientConnectionEvent(event); } }; - + struct TJobResponseMessage { TBusMessage* Request; TBusMessage* Response; EMessageStatus Status; - + TJobResponseMessage(TBusMessage* request, TBusMessage* response, EMessageStatus status) : Request(request) , Response(response) @@ -215,9 +215,9 @@ namespace NBus { public NActor::TQueueInActor<TJobRunner, TJobResponseMessage>, public TScheduleActor<TJobRunner> { THolder<TBusJob> Job; - + TList<TJobRunner*>::iterator JobStorageIterator; - + TJobRunner(TAutoPtr<TBusJob> job) : NActor::TActor<TJobRunner>(job->ModuleImpl->Queue->GetExecutor()) , TScheduleActor<TJobRunner>(&job->ModuleImpl->Scheduler) @@ -226,15 +226,15 @@ namespace NBus { { Job->Runner = this; } - + ~TJobRunner() override { Y_ASSERT(JobStorageIterator == TList<TJobRunner*>::iterator()); } - + void ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, const TJobResponseMessage& message) { Job->CallReplyHandler(message.Status, message.Request, message.Response); } - + void Destroy() { if (!!Job->OnMessageContext) { if (!Job->ReplySent) { @@ -242,24 +242,24 @@ namespace NBus { } } Job->ModuleImpl->DestroyJob(this); - } - + } + void Act(NActor::TDefaultTag) { if (JobStorageIterator == TList<TJobRunner*>::iterator()) { return; } - + if (Job->SleepUntil != 0) { if (AtomicGet(Job->ModuleImpl->State) == TBusModuleImpl::STOPPED) { Destroy(); return; } } - + TThreadCurrentJobGuard g(Job.Get()); - + NActor::TQueueInActor<TJobRunner, TJobResponseMessage>::DequeueAll(); - + if (Alarm.FetchTask()) { if (Job->AnyPendingToSend()) { Y_ASSERT(Job->SleepUntil == 0); @@ -272,50 +272,50 @@ namespace NBus { Y_ASSERT(Job->SleepUntil != 0); Job->SleepUntil = 0; } - } - + } + for (;;) { if (Job->Pending.empty() && !!Job->Handler && Job->Status == MESSAGE_OK) { TWhatThreadDoesPushPop pp("do call job handler (do not confuse with reply handler)"); - + Job->Handler = Job->Handler(Job->Module, Job.Get(), Job->Message); } - + if (Job->SleepUntil != 0) { ScheduleAt(TInstant::MilliSeconds(Job->SleepUntil)); return; } - + Job->SendPending(); - + if (Job->AnyPendingToSend()) { ScheduleAt(TInstant::Now() + TDuration::Seconds(1)); return; } - + if (!Job->Pending.empty()) { // waiting replies return; } - + if (Job->IsDone()) { Destroy(); return; } } - } + } }; - + } - + static inline TJobRunner* GetJob(TBusMessage* message) { return (TJobRunner*)message->Data; - } - + } + static inline void SetJob(TBusMessage* message, TJobRunner* job) { message->Data = job; } - + TBusJob::TBusJob(TBusModule* module, TBusMessage* message) : Status(MESSAGE_OK) , Runner() @@ -346,7 +346,7 @@ namespace NBus { ///////////////////////////////////////////////////////// /// \brief Send messages in pending list - + /// If at least one message is gone return true /// If message has not been send, move it to Finished with appropriate error code bool TBusJob::SendPending() { @@ -358,7 +358,7 @@ namespace NBus { size_t it = 0; while (it != Pending.size()) { TJobState& call = Pending[it]; - + if (call.Status == MESSAGE_DONT_ASK) { EMessageStatus getAddressStatus = MESSAGE_OK; TNetAddr addr; @@ -371,22 +371,22 @@ namespace NBus { if (getAddressStatus == MESSAGE_OK) { // hold extra reference for each request in flight Runner->Ref(); - + if (call.OneWay) { call.Status = call.Session->SendMessageOneWay(call.Message, &addr); } else { call.Status = call.Session->SendMessage(call.Message, &addr); } - + if (call.Status != MESSAGE_OK) { Runner->UnRef(); } - - } else { + + } else { call.Status = getAddressStatus; - } + } } - + if (call.Status == MESSAGE_OK) { ++it; // keep pending list until we get reply } else if (call.Status == MESSAGE_BUSY) { @@ -397,11 +397,11 @@ namespace NBus { DoCallReplyHandler(call); call.Status = MESSAGE_DONT_ASK; call.Message->Reset(); // generate new Id - } else { + } else { Finished.push_back(call); DoCallReplyHandler(call); Pending.erase(Pending.begin() + it); - } + } } return Pending.size() > 0; } @@ -419,12 +419,12 @@ namespace NBus { bool TBusJob::IsDone() { bool r = (SleepUntil == 0 && Pending.size() == 0 && (Handler == nullptr || Status != MESSAGE_OK)); return r; - } - + } + void TBusJob::CallJobHandlerOnly() { TThreadCurrentJobGuard threadCurrentJobGuard(this); TWhatThreadDoesPushPop pp("do call job handler (do not confuse with reply handler)"); - + Handler = Handler(ModuleImpl->Module, this, Message); } @@ -438,31 +438,31 @@ namespace NBus { if (Status != MESSAGE_OK) { break; } - + /// there are messages to send and wait for reply SendPending(); - + if (!Pending.empty()) { break; } - + /// asked to sleep if (SleepUntil) { break; } } - + Y_VERIFY(!(Pending.size() == 0 && Handler == nullptr && Status == MESSAGE_OK && !ReplySent), "Handler returned NULL without Cancel() or SendReply() for message=%016" PRIx64 " type=%d", Message->GetHeader()->Id, Message->GetHeader()->Type); - + return IsDone(); } - + void TBusJob::DoCallReplyHandler(TJobState& call) { if (call.Handler) { TWhatThreadDoesPushPop pp("do call reply handler (do not confuse with job handler)"); - + TThreadCurrentJobGuard threadCurrentJobGuard(this); (Module->*(call.Handler))(this, call.Status, call.Message, call.Reply); } @@ -477,7 +477,7 @@ namespace NBus { break; } } - + /// if not found, report error if (i == Pending.size()) { Y_FAIL("must not happen"); @@ -496,7 +496,7 @@ namespace NBus { DoCallReplyHandler(call); return 0; } - + /// call the handler if provided DoCallReplyHandler(call); @@ -515,50 +515,50 @@ namespace NBus { SetJob(mess.Get(), Runner); Pending.push_back(TJobState(rhandler, MESSAGE_DONT_ASK, mess.Release(), session, nullptr, maxRetries, nullptr, false)); } - + void TBusJob::Send(TBusMessageAutoPtr mess, TBusClientSession* session, TReplyHandler rhandler, size_t maxRetries, const TNetAddr& addr) { CheckThreadCurrentJob(); SetJob(mess.Get(), Runner); Pending.push_back(TJobState(rhandler, MESSAGE_DONT_ASK, mess.Release(), session, nullptr, maxRetries, &addr, false)); } - + void TBusJob::SendOneWayTo(TBusMessageAutoPtr req, TBusClientSession* session, const TNetAddr& addr) { CheckThreadCurrentJob(); SetJob(req.Get(), Runner); Pending.push_back(TJobState(nullptr, MESSAGE_DONT_ASK, req.Release(), session, nullptr, 0, &addr, true)); } - + void TBusJob::SendOneWayWithLocator(TBusMessageAutoPtr req, TBusClientSession* session) { CheckThreadCurrentJob(); - + SetJob(req.Get(), Runner); Pending.push_back(TJobState(nullptr, MESSAGE_DONT_ASK, req.Release(), session, nullptr, 0, nullptr, true)); } - + /////////////////////////////////////////////////////////////// /// send reply to the starter message void TBusJob::SendReply(TBusMessageAutoPtr reply) { CheckThreadCurrentJob(); - + Y_VERIFY(!ReplySent, "cannot call SendReply twice"); ReplySent = true; if (!OnMessageContext) return; - + EMessageStatus ok = OnMessageContext.SendReplyMove(reply); if (ok != MESSAGE_OK) { // TODO: count errors } } - + /// set the flag to terminate job at the earliest convenience void TBusJob::Cancel(EMessageStatus status) { CheckThreadCurrentJob(); - + Status = status; - } + } void TBusJob::ClearState(TJobState& call) { TJobStateVec::iterator it; @@ -580,10 +580,10 @@ namespace NBus { void TBusJob::Sleep(int milliSeconds) { CheckThreadCurrentJob(); - + Y_VERIFY(Pending.empty(), "sleep is not allowed when there are pending job"); Y_VERIFY(SleepUntil == 0, "must not override sleep"); - + SleepUntil = Now() + milliSeconds; } @@ -642,12 +642,12 @@ namespace NBus { : StarterMaxInFlight(1000) { } - + TBusModuleConfig::TSecret::TSecret() : SchedulePeriod(TDuration::Seconds(1)) { } - + TBusModule::TBusModule(const char* name) : Impl(new TBusModuleImpl(this, name)) { @@ -659,16 +659,16 @@ namespace NBus { const char* TBusModule::GetName() const { return Impl->Name; } - + void TBusModule::SetConfig(const TBusModuleConfig& config) { Impl->ModuleConfig = config; } - + bool TBusModule::StartInput() { Y_VERIFY(Impl->State == TBusModuleImpl::CREATED, "state check"); Y_VERIFY(!!Impl->Queue, "state check"); Impl->State = TBusModuleImpl::RUNNING; - + Y_ASSERT(!Impl->ExternalSession); TBusServerSessionPtr extSession = CreateExtSession(*Impl->Queue); if (extSession != nullptr) { @@ -676,14 +676,14 @@ namespace NBus { } return true; - } + } bool TBusModule::Shutdown() { Impl->Shutdown(); return true; } - + TBusJob* TBusModule::CreateJobInstance(TBusMessage* message) { TBusJob* job = new TBusJob(this, message); return job; @@ -706,11 +706,11 @@ TBusSession* TMyModule::CreateExtSession(TBusMessageQueue& queue) { int TBusModule::GetModuleSessionInFlight() const { return Impl->Size(); } - + TIntrusivePtr<TBusModuleInternal> TBusModule::GetInternal() { return Impl.Get(); } - + TBusServerSessionPtr TBusModule::CreateDefaultDestination( TBusMessageQueue& queue, TBusProtocol* proto, const TBusServerSessionConfig& config, const TString& name) { TBusServerSessionConfig patchedConfig = config; @@ -725,7 +725,7 @@ TBusSession* TMyModule::CreateExtSession(TBusMessageQueue& queue) { TBusServerSession::Create(proto, Impl->ModuleServerHandler.Get(), patchedConfig, &queue); Impl->ServerSessions.push_back(session); return session; - } + } TBusClientSessionPtr TBusModule::CreateDefaultSource( TBusMessageQueue& queue, TBusProtocol* proto, const TBusClientSessionConfig& config, const TString& name) { @@ -741,140 +741,140 @@ TBusSession* TMyModule::CreateExtSession(TBusMessageQueue& queue) { TBusClientSession::Create(proto, Impl->ModuleClientHandler.Get(), patchedConfig, &queue); Impl->ClientSessions.push_back(session); return session; - } - + } + TBusStarter* TBusModule::CreateDefaultStarter(TBusMessageQueue&, const TBusSessionConfig& config) { TBusStarter* session = new TBusStarter(this, config); Impl->Starters.push_back(session); return session; - } + } void TBusModule::OnClientConnectionEvent(const TClientConnectionEvent& event) { Y_UNUSED(event); - } - + } + TString TBusModule::GetStatus(unsigned flags) { TString strReturn = Sprintf("%s\n", Impl->Name); strReturn += Impl->GetStatus(flags); return strReturn; } - + } void TBusModuleImpl::AddJob(TJobRunner* jobRunner) { - TWhatThreadDoesAcquireGuard<TMutex> G(Lock, "modules: acquiring lock for AddJob"); - Jobs.push_back(jobRunner); - jobRunner->JobStorageIterator = Jobs.end(); - --jobRunner->JobStorageIterator; -} - + TWhatThreadDoesAcquireGuard<TMutex> G(Lock, "modules: acquiring lock for AddJob"); + Jobs.push_back(jobRunner); + jobRunner->JobStorageIterator = Jobs.end(); + --jobRunner->JobStorageIterator; +} + void TBusModuleImpl::DestroyJob(TJobRunner* job) { Y_ASSERT(job->JobStorageIterator != TList<TJobRunner*>::iterator()); - - { - TWhatThreadDoesAcquireGuard<TMutex> G(Lock, "modules: acquiring lock for DestroyJob"); - int jobCount = AtomicDecrement(JobCount); + + { + TWhatThreadDoesAcquireGuard<TMutex> G(Lock, "modules: acquiring lock for DestroyJob"); + int jobCount = AtomicDecrement(JobCount); Y_VERIFY(jobCount >= 0, "decremented too much"); - Jobs.erase(job->JobStorageIterator); - + Jobs.erase(job->JobStorageIterator); + if (AtomicGet(State) == STOPPED) { - if (jobCount == 0) { - ShutdownCondVar.BroadCast(); - } + if (jobCount == 0) { + ShutdownCondVar.BroadCast(); + } } } - + job->JobStorageIterator = TList<TJobRunner*>::iterator(); } -void TBusModuleImpl::OnMessageReceived(TAutoPtr<TBusMessage> msg0, TOnMessageContext& context) { - TBusMessage* msg = !!msg0 ? msg0.Get() : context.GetMessage(); +void TBusModuleImpl::OnMessageReceived(TAutoPtr<TBusMessage> msg0, TOnMessageContext& context) { + TBusMessage* msg = !!msg0 ? msg0.Get() : context.GetMessage(); Y_VERIFY(!!msg); - - THolder<TJobRunner> jobRunner(new TJobRunner(Module->CreateJobInstance(msg))); - jobRunner->Job->MessageHolder.Reset(msg0.Release()); - jobRunner->Job->OnMessageContext.Swap(context); - SetJob(jobRunner->Job->Message, jobRunner.Get()); - - AtomicIncrement(JobCount); - - AddJob(jobRunner.Get()); - - jobRunner.Release()->Schedule(); -} - -void TBusModuleImpl::Shutdown() { + + THolder<TJobRunner> jobRunner(new TJobRunner(Module->CreateJobInstance(msg))); + jobRunner->Job->MessageHolder.Reset(msg0.Release()); + jobRunner->Job->OnMessageContext.Swap(context); + SetJob(jobRunner->Job->Message, jobRunner.Get()); + + AtomicIncrement(JobCount); + + AddJob(jobRunner.Get()); + + jobRunner.Release()->Schedule(); +} + +void TBusModuleImpl::Shutdown() { if (AtomicGet(State) != TBusModuleImpl::RUNNING) { AtomicSet(State, TBusModuleImpl::STOPPED); - return; - } + return; + } AtomicSet(State, TBusModuleImpl::STOPPED); - + for (auto& clientSession : ClientSessions) { clientSession->Shutdown(); - } + } for (auto& serverSession : ServerSessions) { serverSession->Shutdown(); - } - - for (size_t starter = 0; starter < Starters.size(); ++starter) { - Starters[starter]->Shutdown(); - } - - { - TWhatThreadDoesAcquireGuard<TMutex> guard(Lock, "modules: acquiring lock for Shutdown"); + } + + for (size_t starter = 0; starter < Starters.size(); ++starter) { + Starters[starter]->Shutdown(); + } + + { + TWhatThreadDoesAcquireGuard<TMutex> guard(Lock, "modules: acquiring lock for Shutdown"); for (auto& Job : Jobs) { Job->Schedule(); - } - - while (!Jobs.empty()) { - ShutdownCondVar.WaitI(Lock); - } - } + } + + while (!Jobs.empty()) { + ShutdownCondVar.WaitI(Lock); + } + } } -EMessageStatus TBusModule::StartJob(TAutoPtr<TBusMessage> message) { +EMessageStatus TBusModule::StartJob(TAutoPtr<TBusMessage> message) { Y_VERIFY(Impl->State == TBusModuleImpl::RUNNING); Y_VERIFY(!!Impl->Queue); - + if ((unsigned)AtomicGet(Impl->JobCount) >= Impl->ModuleConfig.StarterMaxInFlight) { - return MESSAGE_BUSY; - } - - TOnMessageContext dummy; - Impl->OnMessageReceived(message.Release(), dummy); - - return MESSAGE_OK; -} - -void TModuleServerHandler::OnMessage(TOnMessageContext& msg) { + return MESSAGE_BUSY; + } + + TOnMessageContext dummy; + Impl->OnMessageReceived(message.Release(), dummy); + + return MESSAGE_OK; +} + +void TModuleServerHandler::OnMessage(TOnMessageContext& msg) { Module->OnMessageReceived(nullptr, msg); -} - -void TModuleClientHandler::OnReply(TAutoPtr<TBusMessage> req, TAutoPtr<TBusMessage> resp) { - TJobRunner* job = GetJob(req.Get()); +} + +void TModuleClientHandler::OnReply(TAutoPtr<TBusMessage> req, TAutoPtr<TBusMessage> resp) { + TJobRunner* job = GetJob(req.Get()); Y_ASSERT(job); Y_ASSERT(job->Job->Message != req.Get()); - job->EnqueueAndSchedule(TJobResponseMessage(req.Release(), resp.Release(), MESSAGE_OK)); - job->UnRef(); + job->EnqueueAndSchedule(TJobResponseMessage(req.Release(), resp.Release(), MESSAGE_OK)); + job->UnRef(); } -void TModuleClientHandler::OnMessageSentOneWay(TAutoPtr<TBusMessage> req) { - TJobRunner* job = GetJob(req.Get()); +void TModuleClientHandler::OnMessageSentOneWay(TAutoPtr<TBusMessage> req) { + TJobRunner* job = GetJob(req.Get()); Y_ASSERT(job); Y_ASSERT(job->Job->Message != req.Get()); job->EnqueueAndSchedule(TJobResponseMessage(req.Release(), nullptr, MESSAGE_OK)); - job->UnRef(); + job->UnRef(); } - -void TModuleClientHandler::OnError(TAutoPtr<TBusMessage> msg, EMessageStatus status) { - TJobRunner* job = GetJob(msg.Get()); - if (job) { + +void TModuleClientHandler::OnError(TAutoPtr<TBusMessage> msg, EMessageStatus status) { + TJobRunner* job = GetJob(msg.Get()); + if (job) { Y_ASSERT(job->Job->Message != msg.Get()); job->EnqueueAndSchedule(TJobResponseMessage(msg.Release(), nullptr, status)); - job->UnRef(); - } -} + job->UnRef(); + } +} void TModuleClientHandler::OnClientConnectionEvent(const TClientConnectionEvent& event) { Module->OnClientConnectionEvent(event); diff --git a/library/cpp/messagebus/oldmodule/module.h b/library/cpp/messagebus/oldmodule/module.h index ead001480c..8d1c4a5d52 100644 --- a/library/cpp/messagebus/oldmodule/module.h +++ b/library/cpp/messagebus/oldmodule/module.h @@ -40,7 +40,7 @@ /// error (not MESSAGE_OK) #include "startsession.h" - + #include <library/cpp/messagebus/ybus.h> #include <util/generic/noncopyable.h> @@ -62,7 +62,7 @@ namespace NBus { protected: typedef TJobHandler (TBusModule::*TBusHandlerPtr)(TBusJob* job, TBusMessage* mess); TBusHandlerPtr MyPtr; - + public: template <class B> TJobHandler(TJobHandler (B::*fptr)(TBusJob* job, TBusMessage* mess)) { @@ -107,7 +107,7 @@ namespace NBus { TNetAddr Addr; bool UseAddr; bool OneWay; - + private: TJobState(TReplyHandler handler, EMessageStatus status, @@ -126,7 +126,7 @@ namespace NBus { Addr = *addr; } UseAddr = !!addr; - } + } public: TString GetStatus(unsigned flags); @@ -140,10 +140,10 @@ namespace NBus { /// Maintains internal state of document in computation class TBusJob { TObjectCounter<TBusJob> ObjectCounter; - + private: void CheckThreadCurrentJob(); - + public: /// given a module and starter message TBusJob(TBusModule* module, TBusMessage* message); @@ -154,9 +154,9 @@ namespace NBus { TBusMessage* GetMessage() const { return Message; } - + TNetAddr GetPeerAddrNetAddr() const; - + /// send message to any other session or application /// If addr is set then use it as destination. void Send(TBusMessageAutoPtr mess, TBusClientSession* session, TReplyHandler rhandler, size_t maxRetries, const TNetAddr& addr); @@ -164,7 +164,7 @@ namespace NBus { void SendOneWayTo(TBusMessageAutoPtr req, TBusClientSession* session, const TNetAddr& addr); void SendOneWayWithLocator(TBusMessageAutoPtr req, TBusClientSession* session); - + /// send reply to the starter message virtual void SendReply(TBusMessageAutoPtr reply); @@ -186,7 +186,7 @@ namespace NBus { Y_ASSERT(stateVec); *stateVec = Pending; } - + /// helper function to find state of previously sent messages template <class MessageType> TJobState* GetState(int* startFrom = nullptr) { @@ -275,18 +275,18 @@ namespace NBus { void Sleep(int milliSeconds); void CallJobHandlerOnly(); - + private: bool CallJobHandler(); void DoCallReplyHandler(TJobState&); /// send out all Pending jobs, failed sends will be migrated to Finished bool SendPending(); bool AnyPendingToSend(); - + public: /// helper to call from OnReply() and OnError() int CallReplyHandler(EMessageStatus status, TBusMessage* mess, TBusMessage* reply); - + public: TJobHandler Handler; ///< job handler to be executed within next CallJobHandler() EMessageStatus Status; ///< set != MESSAGE_OK if job should terminate asap @@ -315,48 +315,48 @@ namespace NBus { //////////////////////////////////////////////////////////////////// /// \brief Classes to implement basic module functionality - + class IJobFactory { protected: virtual ~IJobFactory() { } - + public: /// job factory method, override to create custom jobs virtual TBusJob* CreateJobInstance(TBusMessage* message) = 0; - }; - + }; + struct TBusModuleConfig { unsigned StarterMaxInFlight; - + struct TSecret { TDuration SchedulePeriod; - + TSecret(); }; TSecret Secret; - + TBusModuleConfig(); }; - + namespace NPrivate { struct TBusModuleInternal: public TAtomicRefCount<TBusModuleInternal> { virtual TVector<TBusClientSessionPtr> GetClientSessionsInternal() = 0; virtual TVector<TBusServerSessionPtr> GetServerSessionsInternal() = 0; virtual TBusMessageQueue* GetQueue() = 0; - + virtual TString GetNameInternal() = 0; - + virtual TString GetStatusSingleLine() = 0; - + virtual ~TBusModuleInternal() { } }; } - + class TBusModule: public IJobFactory, TNonCopyable { friend class TBusJob; - + TObjectCounter<TBusModule> ObjectCounter; TIntrusivePtr<NPrivate::TBusModuleImpl> Impl; @@ -365,7 +365,7 @@ namespace NBus { /// Each module should have a name which is used as protocol service TBusModule(const char* name); ~TBusModule() override; - + const char* GetName() const; void SetConfig(const TBusModuleConfig& config); @@ -377,17 +377,17 @@ namespace NBus { virtual bool StartInput(); /// called when application is about to exit virtual bool Shutdown(); - + // this default implementation just creates TBusJob object TBusJob* CreateJobInstance(TBusMessage* message) override; EMessageStatus StartJob(TAutoPtr<TBusMessage> message); - + /// creates private sessions, calls CreateExtSession(), should be called before StartInput() bool CreatePrivateSessions(TBusMessageQueue* queue); - + virtual void OnClientConnectionEvent(const TClientConnectionEvent& event); - + public: /// entry point into module, first function to call virtual TJobHandler Start(TBusJob* job, TBusMessage* mess) = 0; diff --git a/library/cpp/messagebus/oldmodule/startsession.h b/library/cpp/messagebus/oldmodule/startsession.h index 864f82b316..5e26e7e1e5 100644 --- a/library/cpp/messagebus/oldmodule/startsession.h +++ b/library/cpp/messagebus/oldmodule/startsession.h @@ -15,7 +15,7 @@ namespace NBus { bool Exiting; TCondVar ExitSignal; TMutex ExitLock; - + static void* _starter(void* data); void Starter(); diff --git a/library/cpp/messagebus/oldmodule/ya.make b/library/cpp/messagebus/oldmodule/ya.make index 03ff8a46ac..ca5eae74f0 100644 --- a/library/cpp/messagebus/oldmodule/ya.make +++ b/library/cpp/messagebus/oldmodule/ya.make @@ -1,15 +1,15 @@ -LIBRARY() - +LIBRARY() + OWNER(g:messagebus) - -PEERDIR( + +PEERDIR( library/cpp/messagebus library/cpp/messagebus/actor -) - -SRCS( - module.cpp - startsession.cpp -) - -END() +) + +SRCS( + module.cpp + startsession.cpp +) + +END() diff --git a/library/cpp/messagebus/protobuf/ya.make b/library/cpp/messagebus/protobuf/ya.make index 35649705fe..64ff240b51 100644 --- a/library/cpp/messagebus/protobuf/ya.make +++ b/library/cpp/messagebus/protobuf/ya.make @@ -1,15 +1,15 @@ -LIBRARY(messagebus_protobuf) - +LIBRARY(messagebus_protobuf) + OWNER(g:messagebus) - -SRCS( - ybusbuf.cpp -) - -PEERDIR( + +SRCS( + ybusbuf.cpp +) + +PEERDIR( contrib/libs/protobuf library/cpp/messagebus library/cpp/messagebus/actor -) - -END() +) + +END() diff --git a/library/cpp/messagebus/protobuf/ybusbuf.cpp b/library/cpp/messagebus/protobuf/ybusbuf.cpp index e6fe79e2bd..63415b3737 100644 --- a/library/cpp/messagebus/protobuf/ybusbuf.cpp +++ b/library/cpp/messagebus/protobuf/ybusbuf.cpp @@ -1,88 +1,88 @@ #include "ybusbuf.h" - + #include <library/cpp/messagebus/actor/what_thread_does.h> #include <google/protobuf/io/coded_stream.h> - -using namespace NBus; - -TBusBufferProtocol::TBusBufferProtocol(TBusService name, int port) - : TBusProtocol(name, port) -{ -} - -TBusBufferProtocol::~TBusBufferProtocol() { + +using namespace NBus; + +TBusBufferProtocol::TBusBufferProtocol(TBusService name, int port) + : TBusProtocol(name, port) +{ +} + +TBusBufferProtocol::~TBusBufferProtocol() { for (auto& type : Types) { delete type; - } -} - -TBusBufferBase* TBusBufferProtocol::FindType(int type) { - for (unsigned i = 0; i < Types.size(); i++) { - if (Types[i]->GetHeader()->Type == type) { - return Types[i]; - } - } + } +} + +TBusBufferBase* TBusBufferProtocol::FindType(int type) { + for (unsigned i = 0; i < Types.size(); i++) { + if (Types[i]->GetHeader()->Type == type) { + return Types[i]; + } + } return nullptr; -} - -bool TBusBufferProtocol::IsRegisteredType(unsigned type) { - return TypeMask[type >> 5] & (1 << (type & ((1 << 5) - 1))); -} - -void TBusBufferProtocol::RegisterType(TAutoPtr<TBusBufferBase> mess) { - ui32 type = mess->GetHeader()->Type; - TypeMask[type >> 5] |= 1 << (type & ((1 << 5) - 1)); - - Types.push_back(mess.Release()); -} - +} + +bool TBusBufferProtocol::IsRegisteredType(unsigned type) { + return TypeMask[type >> 5] & (1 << (type & ((1 << 5) - 1))); +} + +void TBusBufferProtocol::RegisterType(TAutoPtr<TBusBufferBase> mess) { + ui32 type = mess->GetHeader()->Type; + TypeMask[type >> 5] |= 1 << (type & ((1 << 5) - 1)); + + Types.push_back(mess.Release()); +} + TArrayRef<TBusBufferBase* const> TBusBufferProtocol::GetTypes() const { - return Types; -} - + return Types; +} + void TBusBufferProtocol::Serialize(const TBusMessage* mess, TBuffer& data) { - TWhatThreadDoesPushPop pp("serialize protobuf message"); - + TWhatThreadDoesPushPop pp("serialize protobuf message"); + const TBusHeader* header = mess->GetHeader(); - - if (!IsRegisteredType(header->Type)) { + + if (!IsRegisteredType(header->Type)) { Y_FAIL("unknown message type: %d", int(header->Type)); - return; - } - - // cast the base from real message - const TBusBufferBase* bmess = CheckedCast<const TBusBufferBase*>(mess); - - unsigned size = bmess->GetRecord()->ByteSize(); - data.Reserve(data.Size() + size); - + return; + } + + // cast the base from real message + const TBusBufferBase* bmess = CheckedCast<const TBusBufferBase*>(mess); + + unsigned size = bmess->GetRecord()->ByteSize(); + data.Reserve(data.Size() + size); + char* after = (char*)bmess->GetRecord()->SerializeWithCachedSizesToArray((ui8*)data.Pos()); Y_VERIFY(after - data.Pos() == size); - - data.Advance(size); -} - + + data.Advance(size); +} + TAutoPtr<TBusMessage> TBusBufferProtocol::Deserialize(ui16 messageType, TArrayRef<const char> payload) { - TWhatThreadDoesPushPop pp("deserialize protobuf message"); - + TWhatThreadDoesPushPop pp("deserialize protobuf message"); + TBusBufferBase* messageTemplate = FindType(messageType); if (messageTemplate == nullptr) { return nullptr; //Y_FAIL("unknown message type: %d", unsigned(messageType)); - } - - // clone the base - TAutoPtr<TBusBufferBase> bmess = messageTemplate->New(); - + } + + // clone the base + TAutoPtr<TBusBufferBase> bmess = messageTemplate->New(); + // Need to override protobuf message size limit // NOTE: the payload size has already been checked against session MaxMessageSize google::protobuf::io::CodedInputStream input(reinterpret_cast<const ui8*>(payload.data()), payload.size()); input.SetTotalBytesLimit(payload.size()); bool ok = bmess->GetRecord()->ParseFromCodedStream(&input) && input.ConsumedEntireMessage(); - if (!ok) { + if (!ok) { return nullptr; - } - return bmess.Release(); -} + } + return bmess.Release(); +} diff --git a/library/cpp/messagebus/protobuf/ybusbuf.h b/library/cpp/messagebus/protobuf/ybusbuf.h index bdba972aa3..57b4267ea5 100644 --- a/library/cpp/messagebus/protobuf/ybusbuf.h +++ b/library/cpp/messagebus/protobuf/ybusbuf.h @@ -10,7 +10,7 @@ #include <util/stream/mem.h> #include <array> - + namespace NBus { using TBusBufferRecord = ::google::protobuf::Message; @@ -29,25 +29,25 @@ namespace NBus { : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) { } - + ui16 GetType() const { return GetHeader()->Type; } - + virtual TBusBufferRecord* GetRecord() const = 0; virtual TBusBufferBase* New() = 0; }; - + /////////////////////////////////////////////////////////////////// /// \brief Template for all messages that have protobuf description - + /// @param TBufferRecord is record described in .proto file with namespace /// @param MessageFile is offset for .proto file message ids /// \attention If you want one protocol NBus::TBusBufferProtocol to handle /// messageges described in different .proto files, make sure that they have /// unique values for MessageFile - + template <class TBufferRecord, int MType> class TBusBufferMessage: public TBusBufferBase { public: @@ -93,7 +93,7 @@ namespace NBus { class TBusBufferMessagePtrBase { public: typedef typename TBufferMessage::RecordType RecordType; - + private: TSelf* GetSelf() { return static_cast<TSelf*>(this); @@ -132,7 +132,7 @@ namespace NBus { class TBusBufferMessagePtr: public TBusBufferMessagePtrBase<TBusBufferMessagePtr<TBufferMessage>, TBufferMessage> { protected: TBufferMessage* Holder; - + public: TBusBufferMessagePtr(TBufferMessage* mess) : Holder(mess) @@ -147,14 +147,14 @@ namespace NBus { const TBufferMessage* Get() const { return Holder; } - + operator TBufferMessage*() { return Holder; } operator const TBufferMessage*() const { return Holder; } - + operator TAutoPtr<TBusMessage>() { TAutoPtr<TBusMessage> r(Holder); Holder = 0; @@ -166,12 +166,12 @@ namespace NBus { return r; } }; - + template <class TBufferMessage> class TBusBufferMessageAutoPtr: public TBusBufferMessagePtrBase<TBusBufferMessageAutoPtr<TBufferMessage>, TBufferMessage> { public: TAutoPtr<TBufferMessage> AutoPtr; - + public: TBusBufferMessageAutoPtr() { } @@ -186,11 +186,11 @@ namespace NBus { const TBufferMessage* Get() const { return AutoPtr.Get(); } - + TBufferMessage* Release() const { return AutoPtr.Release(); } - + operator TAutoPtr<TBusMessage>() { return AutoPtr.Release(); } @@ -201,22 +201,22 @@ namespace NBus { ///////////////////////////////////////////// /// \brief Generic protocol object for messages descibed with protobuf - + /// \attention If you mix messages in the same protocol from more than /// .proto file make sure that they have different MessageFile parameter /// in the NBus::TBusBufferMessage template - + class TBusBufferProtocol: public TBusProtocol { private: TVector<TBusBufferBase*> Types; std::array<ui32, ((1 << 16) >> 5)> TypeMask; - + TBusBufferBase* FindType(int type); bool IsRegisteredType(unsigned type); - + public: TBusBufferProtocol(TBusService name, int port); - + ~TBusBufferProtocol() override; /// register all the message that this protocol should handle diff --git a/library/cpp/messagebus/queue_config.cpp b/library/cpp/messagebus/queue_config.cpp index 5d430970fe..78fb52ee49 100644 --- a/library/cpp/messagebus/queue_config.cpp +++ b/library/cpp/messagebus/queue_config.cpp @@ -1,22 +1,22 @@ -#include "queue_config.h" - -using namespace NBus; - -TBusQueueConfig::TBusQueueConfig() { - // workers and listeners configuratioin - NumWorkers = 1; -} - +#include "queue_config.h" + +using namespace NBus; + +TBusQueueConfig::TBusQueueConfig() { + // workers and listeners configuratioin + NumWorkers = 1; +} + void TBusQueueConfig::ConfigureLastGetopt( NLastGetopt::TOpts& opts, const TString& prefix) { opts.AddLongOption(prefix + "worker-count") .RequiredArgument("COUNT") .DefaultValue(ToString(NumWorkers)) .StoreResult(&NumWorkers); -} - +} + TString TBusQueueConfig::PrintToString() const { - TStringStream ss; - ss << "NumWorkers=" << NumWorkers << "\n"; - return ss.Str(); -} + TStringStream ss; + ss << "NumWorkers=" << NumWorkers << "\n"; + return ss.Str(); +} diff --git a/library/cpp/messagebus/queue_config.h b/library/cpp/messagebus/queue_config.h index 2b597c1b01..a9955f0c70 100644 --- a/library/cpp/messagebus/queue_config.h +++ b/library/cpp/messagebus/queue_config.h @@ -1,19 +1,19 @@ -#pragma once - +#pragma once + #include <library/cpp/getopt/last_getopt.h> - -namespace NBus { + +namespace NBus { ////////////////////////////////////////////////////////////////// /// \brief Configuration for message queue struct TBusQueueConfig { TString Name; int NumWorkers; ///< number of threads calling OnMessage(), OnReply() handlers - + TBusQueueConfig(); ///< initializes with default settings - + void ConfigureLastGetopt(NLastGetopt::TOpts&, const TString& prefix = "mb-"); - + TString PrintToString() const; }; - -} + +} diff --git a/library/cpp/messagebus/rain_check/core/coro.cpp b/library/cpp/messagebus/rain_check/core/coro.cpp index eda2fab402..500841dd5b 100644 --- a/library/cpp/messagebus/rain_check/core/coro.cpp +++ b/library/cpp/messagebus/rain_check/core/coro.cpp @@ -1,60 +1,60 @@ #include "coro.h" - -#include "coro_stack.h" - + +#include "coro_stack.h" + #include <util/system/tls.h> #include <util/system/yassert.h> - -using namespace NRainCheck; - + +using namespace NRainCheck; + TContClosure TCoroTaskRunner::ContClosure(TCoroTaskRunner* runner, TArrayRef<char> memRegion) { - TContClosure contClosure; - contClosure.TrampoLine = runner; - contClosure.Stack = memRegion; - return contClosure; -} - -TCoroTaskRunner::TCoroTaskRunner(IEnv* env, ISubtaskListener* parent, TAutoPtr<ICoroTask> impl) - : TTaskRunnerBase(env, parent, impl.Release()) - , Stack(GetImpl()->StackSize) - , ContMachineContext(ContClosure(this, Stack.MemRegion())) - , CoroDone(false) -{ -} - -TCoroTaskRunner::~TCoroTaskRunner() { + TContClosure contClosure; + contClosure.TrampoLine = runner; + contClosure.Stack = memRegion; + return contClosure; +} + +TCoroTaskRunner::TCoroTaskRunner(IEnv* env, ISubtaskListener* parent, TAutoPtr<ICoroTask> impl) + : TTaskRunnerBase(env, parent, impl.Release()) + , Stack(GetImpl()->StackSize) + , ContMachineContext(ContClosure(this, Stack.MemRegion())) + , CoroDone(false) +{ +} + +TCoroTaskRunner::~TCoroTaskRunner() { Y_ASSERT(CoroDone); -} - +} + Y_POD_STATIC_THREAD(TContMachineContext*) CallerContext; Y_POD_STATIC_THREAD(TCoroTaskRunner*) Task; - + bool TCoroTaskRunner::ReplyReceived() { Y_ASSERT(!CoroDone); - - TContMachineContext me; - - CallerContext = &me; - Task = this; - - me.SwitchTo(&ContMachineContext); - - Stack.VerifyNoStackOverflow(); - + + TContMachineContext me; + + CallerContext = &me; + Task = this; + + me.SwitchTo(&ContMachineContext); + + Stack.VerifyNoStackOverflow(); + Y_ASSERT(CallerContext == &me); Y_ASSERT(Task == this); - - return !CoroDone; -} - + + return !CoroDone; +} + void NRainCheck::TCoroTaskRunner::DoRun() { - GetImpl()->Run(); - CoroDone = true; - ContMachineContext.SwitchTo(CallerContext); -} - + GetImpl()->Run(); + CoroDone = true; + ContMachineContext.SwitchTo(CallerContext); +} + void NRainCheck::ICoroTask::WaitForSubtasks() { - Task->ContMachineContext.SwitchTo(CallerContext); -} + Task->ContMachineContext.SwitchTo(CallerContext); +} diff --git a/library/cpp/messagebus/rain_check/core/coro.h b/library/cpp/messagebus/rain_check/core/coro.h index bf2fca54bd..95e2a30f9b 100644 --- a/library/cpp/messagebus/rain_check/core/coro.h +++ b/library/cpp/messagebus/rain_check/core/coro.h @@ -1,58 +1,58 @@ -#pragma once - +#pragma once + #include "coro_stack.h" #include "task.h" #include <util/generic/ptr.h> -#include <util/memory/alloc.h> -#include <util/system/align.h> +#include <util/memory/alloc.h> +#include <util/system/align.h> #include <util/system/context.h> #include <util/system/valgrind.h> - -namespace NRainCheck { - class ICoroTask; - - class TCoroTaskRunner: public TTaskRunnerBase, private ITrampoLine { - friend class ICoroTask; - - private: - NPrivate::TCoroStack Stack; - TContMachineContext ContMachineContext; - bool CoroDone; - - public: - TCoroTaskRunner(IEnv* env, ISubtaskListener* parent, TAutoPtr<ICoroTask> impl); + +namespace NRainCheck { + class ICoroTask; + + class TCoroTaskRunner: public TTaskRunnerBase, private ITrampoLine { + friend class ICoroTask; + + private: + NPrivate::TCoroStack Stack; + TContMachineContext ContMachineContext; + bool CoroDone; + + public: + TCoroTaskRunner(IEnv* env, ISubtaskListener* parent, TAutoPtr<ICoroTask> impl); ~TCoroTaskRunner() override; - - private: + + private: static TContClosure ContClosure(TCoroTaskRunner* runner, TArrayRef<char> memRegion); - + bool ReplyReceived() override /* override */; - + void DoRun() override /* override */; - + ICoroTask* GetImpl() { return (ICoroTask*)GetImplBase(); } - }; - - class ICoroTask: public ITaskBase { - friend class TCoroTaskRunner; - - private: - size_t StackSize; - - public: - typedef TCoroTaskRunner TTaskRunner; - typedef ICoroTask ITask; - + }; + + class ICoroTask: public ITaskBase { + friend class TCoroTaskRunner; + + private: + size_t StackSize; + + public: + typedef TCoroTaskRunner TTaskRunner; + typedef ICoroTask ITask; + ICoroTask(size_t stackSize = 0x2000) : StackSize(stackSize) { } - - virtual void Run() = 0; - static void WaitForSubtasks(); - }; - -} + + virtual void Run() = 0; + static void WaitForSubtasks(); + }; + +} diff --git a/library/cpp/messagebus/rain_check/core/coro_stack.cpp b/library/cpp/messagebus/rain_check/core/coro_stack.cpp index 888d965a23..83b984ca6e 100644 --- a/library/cpp/messagebus/rain_check/core/coro_stack.cpp +++ b/library/cpp/messagebus/rain_check/core/coro_stack.cpp @@ -1,41 +1,41 @@ #include "coro_stack.h" - -#include <util/generic/singleton.h> -#include <util/system/valgrind.h> - + +#include <util/generic/singleton.h> +#include <util/system/valgrind.h> + #include <cstdlib> #include <stdio.h> - -using namespace NRainCheck; -using namespace NRainCheck::NPrivate; - -TCoroStack::TCoroStack(size_t size) - : SizeValue(size) -{ + +using namespace NRainCheck; +using namespace NRainCheck::NPrivate; + +TCoroStack::TCoroStack(size_t size) + : SizeValue(size) +{ Y_VERIFY(size % sizeof(ui32) == 0); Y_VERIFY(size >= 0x1000); - - DataHolder.Reset(malloc(size)); - - // register in valgrind - - *MagicNumberLocation() = MAGIC_NUMBER; - -#if defined(WITH_VALGRIND) + + DataHolder.Reset(malloc(size)); + + // register in valgrind + + *MagicNumberLocation() = MAGIC_NUMBER; + +#if defined(WITH_VALGRIND) ValgrindStackId = VALGRIND_STACK_REGISTER(Data(), (char*)Data() + Size()); -#endif -} - +#endif +} + TCoroStack::~TCoroStack() { -#if defined(WITH_VALGRIND) - VALGRIND_STACK_DEREGISTER(ValgrindStackId); -#endif - - VerifyNoStackOverflow(); -} - -void TCoroStack::FailStackOverflow() { - static const char message[] = "stack overflow\n"; - fputs(message, stderr); - abort(); -} +#if defined(WITH_VALGRIND) + VALGRIND_STACK_DEREGISTER(ValgrindStackId); +#endif + + VerifyNoStackOverflow(); +} + +void TCoroStack::FailStackOverflow() { + static const char message[] = "stack overflow\n"; + fputs(message, stderr); + abort(); +} diff --git a/library/cpp/messagebus/rain_check/core/coro_stack.h b/library/cpp/messagebus/rain_check/core/coro_stack.h index 41ac786470..2f3520e6e4 100644 --- a/library/cpp/messagebus/rain_check/core/coro_stack.h +++ b/library/cpp/messagebus/rain_check/core/coro_stack.h @@ -1,54 +1,54 @@ -#pragma once - +#pragma once + #include <util/generic/array_ref.h> #include <util/generic/ptr.h> -#include <util/system/valgrind.h> - +#include <util/system/valgrind.h> + namespace NRainCheck { namespace NPrivate { struct TCoroStack { THolder<void, TFree> DataHolder; size_t SizeValue; - -#if defined(WITH_VALGRIND) + +#if defined(WITH_VALGRIND) size_t ValgrindStackId; -#endif - +#endif + TCoroStack(size_t size); ~TCoroStack(); - + void* Data() { return DataHolder.Get(); } - + size_t Size() { return SizeValue; } - + TArrayRef<char> MemRegion() { return TArrayRef((char*)Data(), Size()); } - + ui32* MagicNumberLocation() { -#if STACK_GROW_DOWN == 1 +#if STACK_GROW_DOWN == 1 return (ui32*)Data(); -#elif STACK_GROW_DOWN == 0 +#elif STACK_GROW_DOWN == 0 return ((ui32*)(((char*)Data()) + Size())) - 1; -#else -#error "unknown" -#endif +#else +#error "unknown" +#endif } - + static void FailStackOverflow(); - + inline void VerifyNoStackOverflow() noexcept { if (Y_UNLIKELY(*MagicNumberLocation() != MAGIC_NUMBER)) { FailStackOverflow(); } - } - + } + static const ui32 MAGIC_NUMBER = 0xAB4D15FE; }; - + } } diff --git a/library/cpp/messagebus/rain_check/core/coro_ut.cpp b/library/cpp/messagebus/rain_check/core/coro_ut.cpp index 4ee688f4c1..61a33584a5 100644 --- a/library/cpp/messagebus/rain_check/core/coro_ut.cpp +++ b/library/cpp/messagebus/rain_check/core/coro_ut.cpp @@ -1,106 +1,106 @@ #include <library/cpp/testing/unittest/registar.h> - + #include "coro.h" -#include "spawn.h" - +#include "spawn.h" + #include <library/cpp/messagebus/rain_check/test/ut/test.h> - -using namespace NRainCheck; - + +using namespace NRainCheck; + Y_UNIT_TEST_SUITE(RainCheckCoro) { struct TSimpleCoroTask : ICoroTask { - TTestSync* const TestSync; - - TSimpleCoroTask(TTestEnv*, TTestSync* testSync) - : TestSync(testSync) - { - } - + TTestSync* const TestSync; + + TSimpleCoroTask(TTestEnv*, TTestSync* testSync) + : TestSync(testSync) + { + } + void Run() override { - TestSync->WaitForAndIncrement(0); - } - }; - + TestSync->WaitForAndIncrement(0); + } + }; + Y_UNIT_TEST(Simple) { - TTestSync testSync; - - TTestEnv env; - - TIntrusivePtr<TCoroTaskRunner> task = env.SpawnTask<TSimpleCoroTask>(&testSync); - testSync.WaitForAndIncrement(1); - } - + TTestSync testSync; + + TTestEnv env; + + TIntrusivePtr<TCoroTaskRunner> task = env.SpawnTask<TSimpleCoroTask>(&testSync); + testSync.WaitForAndIncrement(1); + } + struct TSleepCoroTask : ICoroTask { - TTestEnv* const Env; - TTestSync* const TestSync; - - TSleepCoroTask(TTestEnv* env, TTestSync* testSync) - : Env(env) - , TestSync(testSync) - { - } - - TSubtaskCompletion SleepCompletion; - + TTestEnv* const Env; + TTestSync* const TestSync; + + TSleepCoroTask(TTestEnv* env, TTestSync* testSync) + : Env(env) + , TestSync(testSync) + { + } + + TSubtaskCompletion SleepCompletion; + void Run() override { - Env->SleepService.Sleep(&SleepCompletion, TDuration::MilliSeconds(1)); - WaitForSubtasks(); - TestSync->WaitForAndIncrement(0); - } - }; - + Env->SleepService.Sleep(&SleepCompletion, TDuration::MilliSeconds(1)); + WaitForSubtasks(); + TestSync->WaitForAndIncrement(0); + } + }; + Y_UNIT_TEST(Sleep) { - TTestSync testSync; - - TTestEnv env; - - TIntrusivePtr<TCoroTaskRunner> task = env.SpawnTask<TSleepCoroTask>(&testSync); - - testSync.WaitForAndIncrement(1); - } - + TTestSync testSync; + + TTestEnv env; + + TIntrusivePtr<TCoroTaskRunner> task = env.SpawnTask<TSleepCoroTask>(&testSync); + + testSync.WaitForAndIncrement(1); + } + struct TSubtask : ICoroTask { - TTestEnv* const Env; - TTestSync* const TestSync; - - TSubtask(TTestEnv* env, TTestSync* testSync) - : Env(env) - , TestSync(testSync) - { - } - + TTestEnv* const Env; + TTestSync* const TestSync; + + TSubtask(TTestEnv* env, TTestSync* testSync) + : Env(env) + , TestSync(testSync) + { + } + void Run() override { - TestSync->CheckAndIncrement(1); - } - }; - + TestSync->CheckAndIncrement(1); + } + }; + struct TSpawnCoroTask : ICoroTask { - TTestEnv* const Env; - TTestSync* const TestSync; - - TSpawnCoroTask(TTestEnv* env, TTestSync* testSync) - : Env(env) - , TestSync(testSync) - { - } - - TSubtaskCompletion SubtaskCompletion; - + TTestEnv* const Env; + TTestSync* const TestSync; + + TSpawnCoroTask(TTestEnv* env, TTestSync* testSync) + : Env(env) + , TestSync(testSync) + { + } + + TSubtaskCompletion SubtaskCompletion; + void Run() override { - TestSync->CheckAndIncrement(0); - SpawnSubtask<TSubtask>(Env, &SubtaskCompletion, TestSync); - WaitForSubtasks(); - TestSync->CheckAndIncrement(2); - } - }; - + TestSync->CheckAndIncrement(0); + SpawnSubtask<TSubtask>(Env, &SubtaskCompletion, TestSync); + WaitForSubtasks(); + TestSync->CheckAndIncrement(2); + } + }; + Y_UNIT_TEST(Spawn) { - TTestSync testSync; - - TTestEnv env; - - TIntrusivePtr<TCoroTaskRunner> task = env.SpawnTask<TSpawnCoroTask>(&testSync); - - testSync.WaitForAndIncrement(3); - } -} + TTestSync testSync; + + TTestEnv env; + + TIntrusivePtr<TCoroTaskRunner> task = env.SpawnTask<TSpawnCoroTask>(&testSync); + + testSync.WaitForAndIncrement(3); + } +} diff --git a/library/cpp/messagebus/rain_check/core/env.cpp b/library/cpp/messagebus/rain_check/core/env.cpp index 150d63d9bb..fdc0000dbd 100644 --- a/library/cpp/messagebus/rain_check/core/env.cpp +++ b/library/cpp/messagebus/rain_check/core/env.cpp @@ -1,3 +1,3 @@ -#include "env.h" - -using namespace NRainCheck; +#include "env.h" + +using namespace NRainCheck; diff --git a/library/cpp/messagebus/rain_check/core/env.h b/library/cpp/messagebus/rain_check/core/env.h index 4e289dbd3d..f6dd7fceb6 100644 --- a/library/cpp/messagebus/rain_check/core/env.h +++ b/library/cpp/messagebus/rain_check/core/env.h @@ -1,47 +1,47 @@ -#pragma once - +#pragma once + #include "sleep.h" #include "spawn.h" - + #include <library/cpp/messagebus/actor/executor.h> - + #include <util/generic/ptr.h> - -namespace NRainCheck { - struct IEnv { - virtual ::NActor::TExecutor* GetExecutor() = 0; + +namespace NRainCheck { + struct IEnv { + virtual ::NActor::TExecutor* GetExecutor() = 0; virtual ~IEnv() { } - }; - - template <typename TSelf> - struct TEnvTemplate: public IEnv { - template <typename TTask, typename TParam> - TIntrusivePtr<typename TTask::TTaskRunner> SpawnTask(TParam param) { + }; + + template <typename TSelf> + struct TEnvTemplate: public IEnv { + template <typename TTask, typename TParam> + TIntrusivePtr<typename TTask::TTaskRunner> SpawnTask(TParam param) { return ::NRainCheck::SpawnTask<TTask, TSelf>((TSelf*)this, param); - } - }; - - template <typename TSelf> - struct TSimpleEnvTemplate: public TEnvTemplate<TSelf> { - ::NActor::TExecutorPtr Executor; - TSleepService SleepService; - - TSimpleEnvTemplate(unsigned threadCount = 0) - : Executor(new ::NActor::TExecutor(threadCount != 0 ? threadCount : 4)) + } + }; + + template <typename TSelf> + struct TSimpleEnvTemplate: public TEnvTemplate<TSelf> { + ::NActor::TExecutorPtr Executor; + TSleepService SleepService; + + TSimpleEnvTemplate(unsigned threadCount = 0) + : Executor(new ::NActor::TExecutor(threadCount != 0 ? threadCount : 4)) { } - + ::NActor::TExecutor* GetExecutor() override { return Executor.Get(); } - }; - - struct TSimpleEnv: public TSimpleEnvTemplate<TSimpleEnv> { + }; + + struct TSimpleEnv: public TSimpleEnvTemplate<TSimpleEnv> { TSimpleEnv(unsigned threadCount = 0) : TSimpleEnvTemplate<TSimpleEnv>(threadCount) { } - }; - -} + }; + +} diff --git a/library/cpp/messagebus/rain_check/core/fwd.h b/library/cpp/messagebus/rain_check/core/fwd.h index 2f8f1d4754..b43ff8c17c 100644 --- a/library/cpp/messagebus/rain_check/core/fwd.h +++ b/library/cpp/messagebus/rain_check/core/fwd.h @@ -1,18 +1,18 @@ -#pragma once - -namespace NRainCheck { - namespace NPrivate { - } - - class ITaskBase; - class ISimpleTask; - class ICoroTask; - - struct ISubtaskListener; - - class TTaskRunnerBase; - - class TSubtaskCompletion; - struct IEnv; - -} +#pragma once + +namespace NRainCheck { + namespace NPrivate { + } + + class ITaskBase; + class ISimpleTask; + class ICoroTask; + + struct ISubtaskListener; + + class TTaskRunnerBase; + + class TSubtaskCompletion; + struct IEnv; + +} diff --git a/library/cpp/messagebus/rain_check/core/rain_check.cpp b/library/cpp/messagebus/rain_check/core/rain_check.cpp index 63bc300554..2ea1f9e21b 100644 --- a/library/cpp/messagebus/rain_check/core/rain_check.cpp +++ b/library/cpp/messagebus/rain_check/core/rain_check.cpp @@ -1 +1 @@ -#include "rain_check.h" +#include "rain_check.h" diff --git a/library/cpp/messagebus/rain_check/core/rain_check.h b/library/cpp/messagebus/rain_check/core/rain_check.h index a97de4537e..0f289717a2 100644 --- a/library/cpp/messagebus/rain_check/core/rain_check.h +++ b/library/cpp/messagebus/rain_check/core/rain_check.h @@ -1,8 +1,8 @@ -#pragma once - -#include "coro.h" +#pragma once + +#include "coro.h" #include "env.h" -#include "simple.h" -#include "sleep.h" -#include "spawn.h" +#include "simple.h" +#include "sleep.h" +#include "spawn.h" #include "task.h" diff --git a/library/cpp/messagebus/rain_check/core/simple.cpp b/library/cpp/messagebus/rain_check/core/simple.cpp index 8dc71a84ee..70182b2f93 100644 --- a/library/cpp/messagebus/rain_check/core/simple.cpp +++ b/library/cpp/messagebus/rain_check/core/simple.cpp @@ -1,18 +1,18 @@ -#include "simple.h" - -using namespace NRainCheck; - -TSimpleTaskRunner::TSimpleTaskRunner(IEnv* env, ISubtaskListener* parentTask, TAutoPtr<ISimpleTask> impl) - : TTaskRunnerBase(env, parentTask, impl.Release()) - , ContinueFunc(&ISimpleTask::Start) -{ -} - -TSimpleTaskRunner::~TSimpleTaskRunner() { +#include "simple.h" + +using namespace NRainCheck; + +TSimpleTaskRunner::TSimpleTaskRunner(IEnv* env, ISubtaskListener* parentTask, TAutoPtr<ISimpleTask> impl) + : TTaskRunnerBase(env, parentTask, impl.Release()) + , ContinueFunc(&ISimpleTask::Start) +{ +} + +TSimpleTaskRunner::~TSimpleTaskRunner() { Y_ASSERT(!ContinueFunc); -} - +} + bool TSimpleTaskRunner::ReplyReceived() { - ContinueFunc = (GetImpl()->*(ContinueFunc.Func))(); - return !!ContinueFunc; -} + ContinueFunc = (GetImpl()->*(ContinueFunc.Func))(); + return !!ContinueFunc; +} diff --git a/library/cpp/messagebus/rain_check/core/simple.h b/library/cpp/messagebus/rain_check/core/simple.h index eeee4c8c23..20e1bf19f5 100644 --- a/library/cpp/messagebus/rain_check/core/simple.h +++ b/library/cpp/messagebus/rain_check/core/simple.h @@ -1,62 +1,62 @@ -#pragma once - -#include "task.h" - -namespace NRainCheck { - class ISimpleTask; - - // Function called on continue - class TContinueFunc { - friend class TSimpleTaskRunner; - - typedef TContinueFunc (ISimpleTask::*TFunc)(); - TFunc Func; - - public: - TContinueFunc() +#pragma once + +#include "task.h" + +namespace NRainCheck { + class ISimpleTask; + + // Function called on continue + class TContinueFunc { + friend class TSimpleTaskRunner; + + typedef TContinueFunc (ISimpleTask::*TFunc)(); + TFunc Func; + + public: + TContinueFunc() : Func(nullptr) { } - - TContinueFunc(void*) + + TContinueFunc(void*) : Func(nullptr) { } - - template <typename TTask> - TContinueFunc(TContinueFunc (TTask::*func)()) + + template <typename TTask> + TContinueFunc(TContinueFunc (TTask::*func)()) : Func((TFunc)func) - { + { static_assert((std::is_base_of<ISimpleTask, TTask>::value), "expect (std::is_base_of<ISimpleTask, TTask>::value)"); - } - - bool operator!() const { - return !Func; - } - }; - - class TSimpleTaskRunner: public TTaskRunnerBase { - public: - TSimpleTaskRunner(IEnv* env, ISubtaskListener* parentTask, TAutoPtr<ISimpleTask>); + } + + bool operator!() const { + return !Func; + } + }; + + class TSimpleTaskRunner: public TTaskRunnerBase { + public: + TSimpleTaskRunner(IEnv* env, ISubtaskListener* parentTask, TAutoPtr<ISimpleTask>); ~TSimpleTaskRunner() override; - - private: - // Function to be called on completion of all pending tasks. - TContinueFunc ContinueFunc; - + + private: + // Function to be called on completion of all pending tasks. + TContinueFunc ContinueFunc; + bool ReplyReceived() override /* override */; - + ISimpleTask* GetImpl() { return (ISimpleTask*)GetImplBase(); } - }; - - class ISimpleTask: public ITaskBase { - public: - typedef TSimpleTaskRunner TTaskRunner; - typedef ISimpleTask ITask; - - virtual TContinueFunc Start() = 0; - }; - -} + }; + + class ISimpleTask: public ITaskBase { + public: + typedef TSimpleTaskRunner TTaskRunner; + typedef ISimpleTask ITask; + + virtual TContinueFunc Start() = 0; + }; + +} diff --git a/library/cpp/messagebus/rain_check/core/simple_ut.cpp b/library/cpp/messagebus/rain_check/core/simple_ut.cpp index 97b5db2d89..d4545e05aa 100644 --- a/library/cpp/messagebus/rain_check/core/simple_ut.cpp +++ b/library/cpp/messagebus/rain_check/core/simple_ut.cpp @@ -1,59 +1,59 @@ #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/messagebus/rain_check/test/ut/test.h> #include <library/cpp/messagebus/latch.h> - + #include <util/system/event.h> - -using namespace NRainCheck; - + +using namespace NRainCheck; + Y_UNIT_TEST_SUITE(RainCheckSimple) { - struct TTaskWithCompletionCallback: public ISimpleTask { - TTestEnv* const Env; - TTestSync* const TestSync; - - TTaskWithCompletionCallback(TTestEnv* env, TTestSync* testSync) - : Env(env) - , TestSync(testSync) + struct TTaskWithCompletionCallback: public ISimpleTask { + TTestEnv* const Env; + TTestSync* const TestSync; + + TTaskWithCompletionCallback(TTestEnv* env, TTestSync* testSync) + : Env(env) + , TestSync(testSync) { } - - TSubtaskCompletion SleepCompletion; - + + TSubtaskCompletion SleepCompletion; + TContinueFunc Start() override { - TestSync->CheckAndIncrement(0); - - Env->SleepService.Sleep(&SleepCompletion, TDuration::MilliSeconds(1)); - SleepCompletion.SetCompletionCallback(&TTaskWithCompletionCallback::SleepCompletionCallback); - - return &TTaskWithCompletionCallback::Last; - } - - void SleepCompletionCallback(TSubtaskCompletion* completion) { + TestSync->CheckAndIncrement(0); + + Env->SleepService.Sleep(&SleepCompletion, TDuration::MilliSeconds(1)); + SleepCompletion.SetCompletionCallback(&TTaskWithCompletionCallback::SleepCompletionCallback); + + return &TTaskWithCompletionCallback::Last; + } + + void SleepCompletionCallback(TSubtaskCompletion* completion) { Y_VERIFY(completion == &SleepCompletion); - TestSync->CheckAndIncrement(1); - - Env->SleepService.Sleep(&SleepCompletion, TDuration::MilliSeconds(1)); - SleepCompletion.SetCompletionCallback(&TTaskWithCompletionCallback::NextSleepCompletionCallback); - } - + TestSync->CheckAndIncrement(1); + + Env->SleepService.Sleep(&SleepCompletion, TDuration::MilliSeconds(1)); + SleepCompletion.SetCompletionCallback(&TTaskWithCompletionCallback::NextSleepCompletionCallback); + } + void NextSleepCompletionCallback(TSubtaskCompletion*) { - TestSync->CheckAndIncrement(2); - } - - TContinueFunc Last() { - TestSync->CheckAndIncrement(3); + TestSync->CheckAndIncrement(2); + } + + TContinueFunc Last() { + TestSync->CheckAndIncrement(3); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(CompletionCallback) { - TTestEnv env; - TTestSync testSync; - - env.SpawnTask<TTaskWithCompletionCallback>(&testSync); - - testSync.WaitForAndIncrement(4); - } -} + TTestEnv env; + TTestSync testSync; + + env.SpawnTask<TTaskWithCompletionCallback>(&testSync); + + testSync.WaitForAndIncrement(4); + } +} diff --git a/library/cpp/messagebus/rain_check/core/sleep.cpp b/library/cpp/messagebus/rain_check/core/sleep.cpp index 10b875bc79..f5d0b4cac9 100644 --- a/library/cpp/messagebus/rain_check/core/sleep.cpp +++ b/library/cpp/messagebus/rain_check/core/sleep.cpp @@ -1,47 +1,47 @@ #include "rain_check.h" -#include <util/system/yassert.h> - -using namespace NRainCheck; -using namespace NRainCheck::NPrivate; -using namespace NBus; -using namespace NBus::NPrivate; - -TSleepService::TSleepService(::NBus::NPrivate::TScheduler* scheduler) - : Scheduler(scheduler) -{ -} - -NRainCheck::TSleepService::TSleepService() - : SchedulerHolder(new TScheduler) - , Scheduler(SchedulerHolder.Get()) -{ -} - +#include <util/system/yassert.h> + +using namespace NRainCheck; +using namespace NRainCheck::NPrivate; +using namespace NBus; +using namespace NBus::NPrivate; + +TSleepService::TSleepService(::NBus::NPrivate::TScheduler* scheduler) + : Scheduler(scheduler) +{ +} + +NRainCheck::TSleepService::TSleepService() + : SchedulerHolder(new TScheduler) + , Scheduler(SchedulerHolder.Get()) +{ +} + NRainCheck::TSleepService::~TSleepService() { - if (!!SchedulerHolder) { - Scheduler->Stop(); - } -} - -namespace { - struct TSleepServiceScheduleItem: public IScheduleItem { - ISubtaskListener* const Parent; - - TSleepServiceScheduleItem(ISubtaskListener* parent, TInstant time) - : IScheduleItem(time) - , Parent(parent) - { - } - + if (!!SchedulerHolder) { + Scheduler->Stop(); + } +} + +namespace { + struct TSleepServiceScheduleItem: public IScheduleItem { + ISubtaskListener* const Parent; + + TSleepServiceScheduleItem(ISubtaskListener* parent, TInstant time) + : IScheduleItem(time) + , Parent(parent) + { + } + void Do() override { - Parent->SetDone(); - } - }; -} - + Parent->SetDone(); + } + }; +} + void TSleepService::Sleep(TSubtaskCompletion* r, TDuration duration) { - TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); - r->SetRunning(current); - Scheduler->Schedule(new TSleepServiceScheduleItem(r, TInstant::Now() + duration)); -} + TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); + r->SetRunning(current); + Scheduler->Schedule(new TSleepServiceScheduleItem(r, TInstant::Now() + duration)); +} diff --git a/library/cpp/messagebus/rain_check/core/sleep.h b/library/cpp/messagebus/rain_check/core/sleep.h index b5b343de98..1a7a1f8674 100644 --- a/library/cpp/messagebus/rain_check/core/sleep.h +++ b/library/cpp/messagebus/rain_check/core/sleep.h @@ -1,24 +1,24 @@ -#pragma once - +#pragma once + #include "fwd.h" - + #include <library/cpp/messagebus/scheduler/scheduler.h> - + #include <util/datetime/base.h> - -namespace NRainCheck { - class TSleepService { - private: - THolder< ::NBus::NPrivate::TScheduler> SchedulerHolder; - ::NBus::NPrivate::TScheduler* const Scheduler; - public: - TSleepService(::NBus::NPrivate::TScheduler*); - TSleepService(); - ~TSleepService(); - - // Wake up a task after given duration. - void Sleep(TSubtaskCompletion* r, TDuration); - }; - -} +namespace NRainCheck { + class TSleepService { + private: + THolder< ::NBus::NPrivate::TScheduler> SchedulerHolder; + ::NBus::NPrivate::TScheduler* const Scheduler; + + public: + TSleepService(::NBus::NPrivate::TScheduler*); + TSleepService(); + ~TSleepService(); + + // Wake up a task after given duration. + void Sleep(TSubtaskCompletion* r, TDuration); + }; + +} diff --git a/library/cpp/messagebus/rain_check/core/sleep_ut.cpp b/library/cpp/messagebus/rain_check/core/sleep_ut.cpp index 3c92fa2ca7..2ae85a87b1 100644 --- a/library/cpp/messagebus/rain_check/core/sleep_ut.cpp +++ b/library/cpp/messagebus/rain_check/core/sleep_ut.cpp @@ -1,46 +1,46 @@ #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/messagebus/rain_check/test/ut/test.h> - + #include <util/system/event.h> -using namespace NRainCheck; -using namespace NActor; - +using namespace NRainCheck; +using namespace NActor; + Y_UNIT_TEST_SUITE(Sleep) { - struct TTestTask: public ISimpleTask { - TSimpleEnv* const Env; - TTestSync* const TestSync; - - TTestTask(TSimpleEnv* env, TTestSync* testSync) - : Env(env) - , TestSync(testSync) + struct TTestTask: public ISimpleTask { + TSimpleEnv* const Env; + TTestSync* const TestSync; + + TTestTask(TSimpleEnv* env, TTestSync* testSync) + : Env(env) + , TestSync(testSync) { } - - TSubtaskCompletion Sleep; - + + TSubtaskCompletion Sleep; + TContinueFunc Start() override { - Env->SleepService.Sleep(&Sleep, TDuration::MilliSeconds(1)); - - TestSync->CheckAndIncrement(0); - - return &TTestTask::Continue; - } - - TContinueFunc Continue() { - TestSync->CheckAndIncrement(1); + Env->SleepService.Sleep(&Sleep, TDuration::MilliSeconds(1)); + + TestSync->CheckAndIncrement(0); + + return &TTestTask::Continue; + } + + TContinueFunc Continue() { + TestSync->CheckAndIncrement(1); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(Test) { - TTestSync testSync; - - TSimpleEnv env; - - TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TTestTask>(&testSync); - - testSync.WaitForAndIncrement(2); - } -} + TTestSync testSync; + + TSimpleEnv env; + + TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TTestTask>(&testSync); + + testSync.WaitForAndIncrement(2); + } +} diff --git a/library/cpp/messagebus/rain_check/core/spawn.cpp b/library/cpp/messagebus/rain_check/core/spawn.cpp index d8fc78c129..c570355fbe 100644 --- a/library/cpp/messagebus/rain_check/core/spawn.cpp +++ b/library/cpp/messagebus/rain_check/core/spawn.cpp @@ -1,5 +1,5 @@ -#include "spawn.h" - +#include "spawn.h" + void NRainCheck::NPrivate::SpawnTaskImpl(TTaskRunnerBase* task) { - task->Schedule(); -} + task->Schedule(); +} diff --git a/library/cpp/messagebus/rain_check/core/spawn.h b/library/cpp/messagebus/rain_check/core/spawn.h index 33ba955e0a..f2b146bf29 100644 --- a/library/cpp/messagebus/rain_check/core/spawn.h +++ b/library/cpp/messagebus/rain_check/core/spawn.h @@ -1,50 +1,50 @@ -#pragma once - -#include "coro.h" -#include "simple.h" +#pragma once + +#include "coro.h" +#include "simple.h" #include "task.h" - -namespace NRainCheck { - namespace NPrivate { - void SpawnTaskImpl(TTaskRunnerBase* task); - - template <typename TTask, typename ITask, typename TRunner, typename TEnv, typename TParam> - TIntrusivePtr<TRunner> SpawnTaskWithRunner(TEnv* env, TParam param1, ISubtaskListener* subtaskListener) { + +namespace NRainCheck { + namespace NPrivate { + void SpawnTaskImpl(TTaskRunnerBase* task); + + template <typename TTask, typename ITask, typename TRunner, typename TEnv, typename TParam> + TIntrusivePtr<TRunner> SpawnTaskWithRunner(TEnv* env, TParam param1, ISubtaskListener* subtaskListener) { static_assert((std::is_base_of<ITask, TTask>::value), "expect (std::is_base_of<ITask, TTask>::value)"); - TIntrusivePtr<TRunner> task(new TRunner(env, subtaskListener, new TTask(env, param1))); - NPrivate::SpawnTaskImpl(task.Get()); - return task; - } - - template <typename TTask, typename ITask, typename TRunner, typename TEnv> - void SpawnSubtaskWithRunner(TEnv* env, TSubtaskCompletion* completion) { + TIntrusivePtr<TRunner> task(new TRunner(env, subtaskListener, new TTask(env, param1))); + NPrivate::SpawnTaskImpl(task.Get()); + return task; + } + + template <typename TTask, typename ITask, typename TRunner, typename TEnv> + void SpawnSubtaskWithRunner(TEnv* env, TSubtaskCompletion* completion) { static_assert((std::is_base_of<ITask, TTask>::value), "expect (std::is_base_of<ITask, TTask>::value)"); - TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); - completion->SetRunning(current); - NPrivate::SpawnTaskImpl(new TRunner(env, completion, new TTask(env))); - } - - template <typename TTask, typename ITask, typename TRunner, typename TEnv, typename TParam> - void SpawnSubtaskWithRunner(TEnv* env, TSubtaskCompletion* completion, TParam param) { + TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); + completion->SetRunning(current); + NPrivate::SpawnTaskImpl(new TRunner(env, completion, new TTask(env))); + } + + template <typename TTask, typename ITask, typename TRunner, typename TEnv, typename TParam> + void SpawnSubtaskWithRunner(TEnv* env, TSubtaskCompletion* completion, TParam param) { static_assert((std::is_base_of<ITask, TTask>::value), "expect (std::is_base_of<ITask, TTask>::value)"); - TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); - completion->SetRunning(current); - NPrivate::SpawnTaskImpl(new TRunner(env, completion, new TTask(env, param))); - } - - } - - // Instantiate and start a task with given parameter. - template <typename TTask, typename TEnv, typename TParam> - TIntrusivePtr<typename TTask::TTaskRunner> SpawnTask(TEnv* env, TParam param1, ISubtaskListener* subtaskListener = &TNopSubtaskListener::Instance) { - return NPrivate::SpawnTaskWithRunner< + TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); + completion->SetRunning(current); + NPrivate::SpawnTaskImpl(new TRunner(env, completion, new TTask(env, param))); + } + + } + + // Instantiate and start a task with given parameter. + template <typename TTask, typename TEnv, typename TParam> + TIntrusivePtr<typename TTask::TTaskRunner> SpawnTask(TEnv* env, TParam param1, ISubtaskListener* subtaskListener = &TNopSubtaskListener::Instance) { + return NPrivate::SpawnTaskWithRunner< TTask, typename TTask::ITask, typename TTask::TTaskRunner, TEnv, TParam>(env, param1, subtaskListener); - } - - // Instantiate and start subtask of given task. - template <typename TTask, typename TEnv, typename TParam> - void SpawnSubtask(TEnv* env, TSubtaskCompletion* completion, TParam param) { - return NPrivate::SpawnSubtaskWithRunner<TTask, typename TTask::ITask, typename TTask::TTaskRunner>(env, completion, param); - } - -} + } + + // Instantiate and start subtask of given task. + template <typename TTask, typename TEnv, typename TParam> + void SpawnSubtask(TEnv* env, TSubtaskCompletion* completion, TParam param) { + return NPrivate::SpawnSubtaskWithRunner<TTask, typename TTask::ITask, typename TTask::TTaskRunner>(env, completion, param); + } + +} diff --git a/library/cpp/messagebus/rain_check/core/spawn_ut.cpp b/library/cpp/messagebus/rain_check/core/spawn_ut.cpp index 2b3ef75c67..ba5a5e41cf 100644 --- a/library/cpp/messagebus/rain_check/core/spawn_ut.cpp +++ b/library/cpp/messagebus/rain_check/core/spawn_ut.cpp @@ -1,145 +1,145 @@ #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/messagebus/rain_check/test/helper/misc.h> #include <library/cpp/messagebus/rain_check/test/ut/test.h> #include <library/cpp/messagebus/latch.h> - + #include <util/system/event.h> - + #include <array> - -using namespace NRainCheck; -using namespace NActor; - + +using namespace NRainCheck; +using namespace NActor; + Y_UNIT_TEST_SUITE(Spawn) { - struct TTestTask: public ISimpleTask { - TTestSync* const TestSync; - - TTestTask(TSimpleEnv*, TTestSync* testSync) - : TestSync(testSync) - , I(0) + struct TTestTask: public ISimpleTask { + TTestSync* const TestSync; + + TTestTask(TSimpleEnv*, TTestSync* testSync) + : TestSync(testSync) + , I(0) { } - + TSystemEvent Started; - - unsigned I; - + + unsigned I; + TContinueFunc Start() override { - if (I < 4) { - I += 1; - return &TTestTask::Start; - } - TestSync->CheckAndIncrement(0); - return &TTestTask::Continue; - } - - TContinueFunc Continue() { - TestSync->CheckAndIncrement(1); - - Started.Signal(); + if (I < 4) { + I += 1; + return &TTestTask::Start; + } + TestSync->CheckAndIncrement(0); + return &TTestTask::Continue; + } + + TContinueFunc Continue() { + TestSync->CheckAndIncrement(1); + + Started.Signal(); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(Continuation) { - TTestSync testSync; - - TSimpleEnv env; - - TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TTestTask>(&testSync); - - testSync.WaitForAndIncrement(2); - } - - struct TSubtask: public ISimpleTask { - TTestEnv* const Env; - TTestSync* const TestSync; - - TSubtask(TTestEnv* env, TTestSync* testSync) - : Env(env) - , TestSync(testSync) + TTestSync testSync; + + TSimpleEnv env; + + TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TTestTask>(&testSync); + + testSync.WaitForAndIncrement(2); + } + + struct TSubtask: public ISimpleTask { + TTestEnv* const Env; + TTestSync* const TestSync; + + TSubtask(TTestEnv* env, TTestSync* testSync) + : Env(env) + , TestSync(testSync) { } - + TContinueFunc Start() override { - Sleep(TDuration::MilliSeconds(1)); - TestSync->CheckAndIncrement(1); + Sleep(TDuration::MilliSeconds(1)); + TestSync->CheckAndIncrement(1); return nullptr; - } - }; - - struct TSpawnTask: public ISimpleTask { - TTestEnv* const Env; - TTestSync* const TestSync; - - TSpawnTask(TTestEnv* env, TTestSync* testSync) - : Env(env) - , TestSync(testSync) + } + }; + + struct TSpawnTask: public ISimpleTask { + TTestEnv* const Env; + TTestSync* const TestSync; + + TSpawnTask(TTestEnv* env, TTestSync* testSync) + : Env(env) + , TestSync(testSync) { } - - TSubtaskCompletion SubtaskCompletion; - + + TSubtaskCompletion SubtaskCompletion; + TContinueFunc Start() override { - TestSync->CheckAndIncrement(0); - SpawnSubtask<TSubtask>(Env, &SubtaskCompletion, TestSync); - return &TSpawnTask::Continue; - } - - TContinueFunc Continue() { - TestSync->CheckAndIncrement(2); + TestSync->CheckAndIncrement(0); + SpawnSubtask<TSubtask>(Env, &SubtaskCompletion, TestSync); + return &TSpawnTask::Continue; + } + + TContinueFunc Continue() { + TestSync->CheckAndIncrement(2); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(Subtask) { - TTestSync testSync; - - TTestEnv env; - - TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TSpawnTask>(&testSync); - - testSync.WaitForAndIncrement(3); - } - - struct TSpawnLongTask: public ISimpleTask { - TTestEnv* const Env; - TTestSync* const TestSync; - unsigned I; - + TTestSync testSync; + + TTestEnv env; + + TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TSpawnTask>(&testSync); + + testSync.WaitForAndIncrement(3); + } + + struct TSpawnLongTask: public ISimpleTask { + TTestEnv* const Env; + TTestSync* const TestSync; + unsigned I; + TSpawnLongTask(TTestEnv* env, TTestSync* testSync) : Env(env) , TestSync(testSync) , I(0) { } - + std::array<TSubtaskCompletion, 3> Subtasks; - + TContinueFunc Start() override { - if (I == 1000) { - TestSync->CheckAndIncrement(0); + if (I == 1000) { + TestSync->CheckAndIncrement(0); return nullptr; - } - + } + for (auto& subtask : Subtasks) { SpawnSubtask<TNopSimpleTask>(Env, &subtask, ""); - } - - ++I; - return &TSpawnLongTask::Start; - } - }; - + } + + ++I; + return &TSpawnLongTask::Start; + } + }; + Y_UNIT_TEST(SubtaskLong) { - TTestSync testSync; - - TTestEnv env; - - TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TSpawnLongTask>(&testSync); - - testSync.WaitForAndIncrement(1); - } -} + TTestSync testSync; + + TTestEnv env; + + TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TSpawnLongTask>(&testSync); + + testSync.WaitForAndIncrement(1); + } +} diff --git a/library/cpp/messagebus/rain_check/core/task.cpp b/library/cpp/messagebus/rain_check/core/task.cpp index d20ae30402..a098437d53 100644 --- a/library/cpp/messagebus/rain_check/core/task.cpp +++ b/library/cpp/messagebus/rain_check/core/task.cpp @@ -1,216 +1,216 @@ #include "rain_check.h" - + #include <library/cpp/messagebus/actor/temp_tls_vector.h> - + #include <util/system/type_name.h> #include <util/system/tls.h> - -using namespace NRainCheck; -using namespace NRainCheck::NPrivate; - -using namespace NActor; - -namespace { + +using namespace NRainCheck; +using namespace NRainCheck::NPrivate; + +using namespace NActor; + +namespace { Y_POD_STATIC_THREAD(TTaskRunnerBase*) ThreadCurrentTask; -} - +} + void TNopSubtaskListener::SetDone() { } - -TNopSubtaskListener TNopSubtaskListener::Instance; - -TTaskRunnerBase::TTaskRunnerBase(IEnv* env, ISubtaskListener* parentTask, TAutoPtr<ITaskBase> impl) - : TActor<TTaskRunnerBase>(env->GetExecutor()) - , Impl(impl) - , ParentTask(parentTask) - //, HoldsSelfReference(false) - , Done(false) - , SetDoneCalled(false) -{ -} - -TTaskRunnerBase::~TTaskRunnerBase() { + +TNopSubtaskListener TNopSubtaskListener::Instance; + +TTaskRunnerBase::TTaskRunnerBase(IEnv* env, ISubtaskListener* parentTask, TAutoPtr<ITaskBase> impl) + : TActor<TTaskRunnerBase>(env->GetExecutor()) + , Impl(impl) + , ParentTask(parentTask) + //, HoldsSelfReference(false) + , Done(false) + , SetDoneCalled(false) +{ +} + +TTaskRunnerBase::~TTaskRunnerBase() { Y_ASSERT(Done); -} - -namespace { - struct TRunningInThisThreadGuard { - TTaskRunnerBase* const Task; - TRunningInThisThreadGuard(TTaskRunnerBase* task) - : Task(task) - { +} + +namespace { + struct TRunningInThisThreadGuard { + TTaskRunnerBase* const Task; + TRunningInThisThreadGuard(TTaskRunnerBase* task) + : Task(task) + { Y_ASSERT(!ThreadCurrentTask); - ThreadCurrentTask = task; - } - - ~TRunningInThisThreadGuard() { + ThreadCurrentTask = task; + } + + ~TRunningInThisThreadGuard() { Y_ASSERT(ThreadCurrentTask == Task); ThreadCurrentTask = nullptr; - } - }; -} - + } + }; +} + void NRainCheck::TTaskRunnerBase::Act(NActor::TDefaultTag) { Y_ASSERT(RefCount() > 0); - - TRunningInThisThreadGuard g(this); - - //RetainRef(); - - for (;;) { - TTempTlsVector<TSubtaskCompletion*> temp; - - temp.GetVector()->swap(Pending); - + + TRunningInThisThreadGuard g(this); + + //RetainRef(); + + for (;;) { + TTempTlsVector<TSubtaskCompletion*> temp; + + temp.GetVector()->swap(Pending); + for (auto& pending : *temp.GetVector()) { if (pending->IsComplete()) { pending->FireCompletionCallback(GetImplBase()); - } else { + } else { Pending.push_back(pending); - } - } - - if (!Pending.empty()) { - return; - } - - if (!Done) { - Done = !ReplyReceived(); - } else { - if (Pending.empty()) { - if (!SetDoneCalled) { - ParentTask->SetDone(); - SetDoneCalled = true; - } - //ReleaseRef(); - return; - } - } - } -} - + } + } + + if (!Pending.empty()) { + return; + } + + if (!Done) { + Done = !ReplyReceived(); + } else { + if (Pending.empty()) { + if (!SetDoneCalled) { + ParentTask->SetDone(); + SetDoneCalled = true; + } + //ReleaseRef(); + return; + } + } + } +} + bool TTaskRunnerBase::IsRunningInThisThread() const { - return ThreadCurrentTask == this; -} - + return ThreadCurrentTask == this; +} + TSubtaskCompletion::~TSubtaskCompletion() { - ESubtaskState state = State.Get(); + ESubtaskState state = State.Get(); Y_ASSERT(state == CREATED || state == DONE || state == CANCELED); -} - +} + void TSubtaskCompletion::FireCompletionCallback(ITaskBase* task) { Y_ASSERT(IsComplete()); - - if (!!CompletionFunc) { - TSubtaskCompletionFunc temp = CompletionFunc; - // completion func must be reset before calling it, - // because function may set it back - CompletionFunc = TSubtaskCompletionFunc(); - (task->*(temp.Func))(this); - } -} - + + if (!!CompletionFunc) { + TSubtaskCompletionFunc temp = CompletionFunc; + // completion func must be reset before calling it, + // because function may set it back + CompletionFunc = TSubtaskCompletionFunc(); + (task->*(temp.Func))(this); + } +} + void NRainCheck::TSubtaskCompletion::Cancel() { - for (;;) { - ESubtaskState state = State.Get(); - if (state == CREATED && State.CompareAndSet(CREATED, CANCELED)) { - return; - } - if (state == RUNNING && State.CompareAndSet(RUNNING, CANCEL_REQUESTED)) { - return; - } - if (state == DONE && State.CompareAndSet(DONE, CANCELED)) { - return; - } - if (state == CANCEL_REQUESTED || state == CANCELED) { - return; - } - } -} - + for (;;) { + ESubtaskState state = State.Get(); + if (state == CREATED && State.CompareAndSet(CREATED, CANCELED)) { + return; + } + if (state == RUNNING && State.CompareAndSet(RUNNING, CANCEL_REQUESTED)) { + return; + } + if (state == DONE && State.CompareAndSet(DONE, CANCELED)) { + return; + } + if (state == CANCEL_REQUESTED || state == CANCELED) { + return; + } + } +} + void TSubtaskCompletion::SetRunning(TTaskRunnerBase* parent) { Y_ASSERT(!TaskRunner); Y_ASSERT(!!parent); - - TaskRunner = parent; - - parent->Pending.push_back(this); - - parent->RefV(); - - for (;;) { - ESubtaskState current = State.Get(); - if (current != CREATED && current != DONE) { + + TaskRunner = parent; + + parent->Pending.push_back(this); + + parent->RefV(); + + for (;;) { + ESubtaskState current = State.Get(); + if (current != CREATED && current != DONE) { Y_FAIL("current state should be CREATED or DONE: %s", ToCString(current)); - } - if (State.CompareAndSet(current, RUNNING)) { - return; - } - } -} - + } + if (State.CompareAndSet(current, RUNNING)) { + return; + } + } +} + void TSubtaskCompletion::SetDone() { Y_ASSERT(!!TaskRunner); - TTaskRunnerBase* temp = TaskRunner; + TTaskRunnerBase* temp = TaskRunner; TaskRunner = nullptr; - - for (;;) { - ESubtaskState state = State.Get(); - if (state == RUNNING) { - if (State.CompareAndSet(RUNNING, DONE)) { - break; - } - } else if (state == CANCEL_REQUESTED) { - if (State.CompareAndSet(CANCEL_REQUESTED, CANCELED)) { - break; - } - } else { + + for (;;) { + ESubtaskState state = State.Get(); + if (state == RUNNING) { + if (State.CompareAndSet(RUNNING, DONE)) { + break; + } + } else if (state == CANCEL_REQUESTED) { + if (State.CompareAndSet(CANCEL_REQUESTED, CANCELED)) { + break; + } + } else { Y_FAIL("cannot SetDone: unknown state: %s", ToCString(state)); - } - } - - temp->ScheduleV(); - temp->UnRefV(); -} - -#if 0 -void NRainCheck::TTaskRunnerBase::RetainRef() -{ - if (HoldsSelfReference) { - return; - } - HoldsSelfReference = true; - Ref(); -} - -void NRainCheck::TTaskRunnerBase::ReleaseRef() -{ - if (!HoldsSelfReference) { - return; - } - HoldsSelfReference = false; - DecRef(); -} -#endif - + } + } + + temp->ScheduleV(); + temp->UnRefV(); +} + +#if 0 +void NRainCheck::TTaskRunnerBase::RetainRef() +{ + if (HoldsSelfReference) { + return; + } + HoldsSelfReference = true; + Ref(); +} + +void NRainCheck::TTaskRunnerBase::ReleaseRef() +{ + if (!HoldsSelfReference) { + return; + } + HoldsSelfReference = false; + DecRef(); +} +#endif + void TTaskRunnerBase::AssertInThisThread() const { Y_ASSERT(IsRunningInThisThread()); -} - +} + TTaskRunnerBase* TTaskRunnerBase::CurrentTask() { Y_VERIFY(!!ThreadCurrentTask); - return ThreadCurrentTask; -} - + return ThreadCurrentTask; +} + ITaskBase* TTaskRunnerBase::CurrentTaskImpl() { return CurrentTask()->GetImplBase(); } TString TTaskRunnerBase::GetStatusSingleLine() { return TypeName(*Impl); -} - +} + bool NRainCheck::AreWeInsideTask() { return ThreadCurrentTask != nullptr; } diff --git a/library/cpp/messagebus/rain_check/core/task.h b/library/cpp/messagebus/rain_check/core/task.h index b84e62a1eb..7d8778bcda 100644 --- a/library/cpp/messagebus/rain_check/core/task.h +++ b/library/cpp/messagebus/rain_check/core/task.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include "fwd.h" #include <library/cpp/messagebus/actor/actor.h> @@ -7,55 +7,55 @@ #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> -#include <util/generic/noncopyable.h> -#include <util/generic/ptr.h> -#include <util/thread/lfstack.h> - -namespace NRainCheck { - struct ISubtaskListener { - virtual void SetDone() = 0; +#include <util/generic/noncopyable.h> +#include <util/generic/ptr.h> +#include <util/thread/lfstack.h> + +namespace NRainCheck { + struct ISubtaskListener { + virtual void SetDone() = 0; virtual ~ISubtaskListener() { } - }; - - struct TNopSubtaskListener: public ISubtaskListener { + }; + + struct TNopSubtaskListener: public ISubtaskListener { void SetDone() override; - - static TNopSubtaskListener Instance; - }; - - class TSubtaskCompletionFunc { - friend class TSubtaskCompletion; - - typedef void (ITaskBase::*TFunc)(TSubtaskCompletion*); - TFunc Func; - - public: - TSubtaskCompletionFunc() + + static TNopSubtaskListener Instance; + }; + + class TSubtaskCompletionFunc { + friend class TSubtaskCompletion; + + typedef void (ITaskBase::*TFunc)(TSubtaskCompletion*); + TFunc Func; + + public: + TSubtaskCompletionFunc() : Func(nullptr) { } - - TSubtaskCompletionFunc(void*) + + TSubtaskCompletionFunc(void*) : Func(nullptr) { } - - template <typename TTask> - TSubtaskCompletionFunc(void (TTask::*func)(TSubtaskCompletion*)) + + template <typename TTask> + TSubtaskCompletionFunc(void (TTask::*func)(TSubtaskCompletion*)) : Func((TFunc)func) - { + { static_assert((std::is_base_of<ITaskBase, TTask>::value), "expect (std::is_base_of<ITaskBase, TTask>::value)"); - } - - bool operator!() const { - return !Func; - } - }; - - template <typename T> - class TTaskFuture; - + } + + bool operator!() const { + return !Func; + } + }; + + template <typename T> + class TTaskFuture; + #define SUBTASK_STATE_MAP(XX) \ XX(CREATED, "Initial") \ XX(RUNNING, "Running") \ @@ -63,33 +63,33 @@ namespace NRainCheck { XX(CANCEL_REQUESTED, "Cancel requested, but still executing") \ XX(CANCELED, "Canceled") \ /**/ - - enum ESubtaskState { - SUBTASK_STATE_MAP(ENUM_VALUE_GEN_NO_VALUE) - }; - - ENUM_TO_STRING(ESubtaskState, SUBTASK_STATE_MAP) - + + enum ESubtaskState { + SUBTASK_STATE_MAP(ENUM_VALUE_GEN_NO_VALUE) + }; + + ENUM_TO_STRING(ESubtaskState, SUBTASK_STATE_MAP) + class TSubtaskCompletion : TNonCopyable, public ISubtaskListener { - friend struct TTaskAccessor; + friend struct TTaskAccessor; - private: - TAtomicBox<ESubtaskState> State; - TTaskRunnerBase* volatile TaskRunner; - TSubtaskCompletionFunc CompletionFunc; + private: + TAtomicBox<ESubtaskState> State; + TTaskRunnerBase* volatile TaskRunner; + TSubtaskCompletionFunc CompletionFunc; - public: + public: TSubtaskCompletion() : State(CREATED) , TaskRunner() { } ~TSubtaskCompletion() override; - - // Either done or cancel requested or cancelled - bool IsComplete() const { - ESubtaskState state = State.Get(); - switch (state) { + + // Either done or cancel requested or cancelled + bool IsComplete() const { + ESubtaskState state = State.Get(); + switch (state) { case RUNNING: return false; case DONE: @@ -102,82 +102,82 @@ namespace NRainCheck { Y_FAIL("not started"); default: Y_FAIL("unknown value: %u", (unsigned)state); - } - } - - void FireCompletionCallback(ITaskBase*); - - void SetCompletionCallback(TSubtaskCompletionFunc func) { - CompletionFunc = func; - } - - // Completed, but not cancelled - bool IsDone() const { - return State.Get() == DONE; - } - - // Request cancel by actor - // Does nothing but marks task cancelled, - // and allows proceeding to next callback - void Cancel(); - - // called by service provider implementations - // must not be called by actor - void SetRunning(TTaskRunnerBase* parent); + } + } + + void FireCompletionCallback(ITaskBase*); + + void SetCompletionCallback(TSubtaskCompletionFunc func) { + CompletionFunc = func; + } + + // Completed, but not cancelled + bool IsDone() const { + return State.Get() == DONE; + } + + // Request cancel by actor + // Does nothing but marks task cancelled, + // and allows proceeding to next callback + void Cancel(); + + // called by service provider implementations + // must not be called by actor + void SetRunning(TTaskRunnerBase* parent); void SetDone() override; - }; - - // See ISimpleTask, ICoroTask + }; + + // See ISimpleTask, ICoroTask class TTaskRunnerBase: public TAtomicRefCount<TTaskRunnerBase>, public NActor::TActor<TTaskRunnerBase> { - friend class NActor::TActor<TTaskRunnerBase>; - friend class TContinueFunc; - friend struct TTaskAccessor; - friend class TSubtaskCompletion; - - private: - THolder<ITaskBase> Impl; - - ISubtaskListener* const ParentTask; - // While task is running, it holds extra reference to self. - //bool HoldsSelfReference; - bool Done; - bool SetDoneCalled; - - // Subtasks currently executed. + friend class NActor::TActor<TTaskRunnerBase>; + friend class TContinueFunc; + friend struct TTaskAccessor; + friend class TSubtaskCompletion; + + private: + THolder<ITaskBase> Impl; + + ISubtaskListener* const ParentTask; + // While task is running, it holds extra reference to self. + //bool HoldsSelfReference; + bool Done; + bool SetDoneCalled; + + // Subtasks currently executed. TVector<TSubtaskCompletion*> Pending; - - void Act(NActor::TDefaultTag); - - public: - // Construct task. Task is not automatically started. - TTaskRunnerBase(IEnv*, ISubtaskListener* parent, TAutoPtr<ITaskBase> impl); + + void Act(NActor::TDefaultTag); + + public: + // Construct task. Task is not automatically started. + TTaskRunnerBase(IEnv*, ISubtaskListener* parent, TAutoPtr<ITaskBase> impl); ~TTaskRunnerBase() override; - - bool IsRunningInThisThread() const; - void AssertInThisThread() const; - static TTaskRunnerBase* CurrentTask(); + + bool IsRunningInThisThread() const; + void AssertInThisThread() const; + static TTaskRunnerBase* CurrentTask(); static ITaskBase* CurrentTaskImpl(); - + TString GetStatusSingleLine(); - - protected: - //void RetainRef(); - //void ReleaseRef(); + + protected: + //void RetainRef(); + //void ReleaseRef(); ITaskBase* GetImplBase() { return Impl.Get(); } - - private: - // true if need to call again - virtual bool ReplyReceived() = 0; - }; - - class ITaskBase { - public: + + private: + // true if need to call again + virtual bool ReplyReceived() = 0; + }; + + class ITaskBase { + public: virtual ~ITaskBase() { } - }; - + }; + // Check that current method executed inside some task. bool AreWeInsideTask(); diff --git a/library/cpp/messagebus/rain_check/core/track.cpp b/library/cpp/messagebus/rain_check/core/track.cpp index cc3747b9f6..092a51a214 100644 --- a/library/cpp/messagebus/rain_check/core/track.cpp +++ b/library/cpp/messagebus/rain_check/core/track.cpp @@ -1,66 +1,66 @@ -#include "track.h" - -using namespace NRainCheck; -using namespace NRainCheck::NPrivate; - +#include "track.h" + +using namespace NRainCheck; +using namespace NRainCheck::NPrivate; + void TTaskTrackerReceipt::SetDone() { - TaskTracker->GetQueue<TTaskTrackerReceipt*>()->EnqueueAndSchedule(this); -} - + TaskTracker->GetQueue<TTaskTrackerReceipt*>()->EnqueueAndSchedule(this); +} + TString TTaskTrackerReceipt::GetStatusSingleLine() { - return Task->GetStatusSingleLine(); -} - + return Task->GetStatusSingleLine(); +} + TTaskTracker::TTaskTracker(NActor::TExecutor* executor) : NActor::TActor<TTaskTracker>(executor) -{ -} - +{ +} + TTaskTracker::~TTaskTracker() { Y_ASSERT(Tasks.Empty()); -} - -void TTaskTracker::Shutdown() { - ShutdownFlag.Set(true); - Schedule(); - ShutdownEvent.WaitI(); -} - -void TTaskTracker::ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, ITaskFactory* taskFactory) { - THolder<ITaskFactory> holder(taskFactory); - - THolder<TTaskTrackerReceipt> receipt(new TTaskTrackerReceipt(this)); - receipt->Task = taskFactory->NewTask(receipt.Get()); - - Tasks.PushBack(receipt.Release()); -} - -void TTaskTracker::ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, TTaskTrackerReceipt* receipt) { +} + +void TTaskTracker::Shutdown() { + ShutdownFlag.Set(true); + Schedule(); + ShutdownEvent.WaitI(); +} + +void TTaskTracker::ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, ITaskFactory* taskFactory) { + THolder<ITaskFactory> holder(taskFactory); + + THolder<TTaskTrackerReceipt> receipt(new TTaskTrackerReceipt(this)); + receipt->Task = taskFactory->NewTask(receipt.Get()); + + Tasks.PushBack(receipt.Release()); +} + +void TTaskTracker::ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, TTaskTrackerReceipt* receipt) { Y_ASSERT(!receipt->Empty()); - receipt->Unlink(); - delete receipt; -} - -void TTaskTracker::ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, TAsyncResult<TTaskTrackerStatus>* status) { - TTaskTrackerStatus s; - s.Size = Tasks.Size(); - status->SetResult(s); -} - + receipt->Unlink(); + delete receipt; +} + +void TTaskTracker::ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, TAsyncResult<TTaskTrackerStatus>* status) { + TTaskTrackerStatus s; + s.Size = Tasks.Size(); + status->SetResult(s); +} + void TTaskTracker::Act(NActor::TDefaultTag) { - GetQueue<TAsyncResult<TTaskTrackerStatus>*>()->DequeueAll(); - GetQueue<ITaskFactory*>()->DequeueAll(); - GetQueue<TTaskTrackerReceipt*>()->DequeueAll(); - - if (ShutdownFlag.Get()) { - if (Tasks.Empty()) { - ShutdownEvent.Signal(); - } - } -} - -ui32 TTaskTracker::Size() { - TAsyncResult<TTaskTrackerStatus> r; - GetQueue<TAsyncResult<TTaskTrackerStatus>*>()->EnqueueAndSchedule(&r); - return r.GetResult().Size; -} + GetQueue<TAsyncResult<TTaskTrackerStatus>*>()->DequeueAll(); + GetQueue<ITaskFactory*>()->DequeueAll(); + GetQueue<TTaskTrackerReceipt*>()->DequeueAll(); + + if (ShutdownFlag.Get()) { + if (Tasks.Empty()) { + ShutdownEvent.Signal(); + } + } +} + +ui32 TTaskTracker::Size() { + TAsyncResult<TTaskTrackerStatus> r; + GetQueue<TAsyncResult<TTaskTrackerStatus>*>()->EnqueueAndSchedule(&r); + return r.GetResult().Size; +} diff --git a/library/cpp/messagebus/rain_check/core/track.h b/library/cpp/messagebus/rain_check/core/track.h index a7f3d099f0..d387de7574 100644 --- a/library/cpp/messagebus/rain_check/core/track.h +++ b/library/cpp/messagebus/rain_check/core/track.h @@ -1,97 +1,97 @@ -#pragma once - +#pragma once + #include "spawn.h" #include "task.h" - + #include <library/cpp/messagebus/async_result.h> #include <library/cpp/messagebus/actor/queue_in_actor.h> #include <library/cpp/messagebus/misc/atomic_box.h> - + #include <util/generic/intrlist.h> #include <util/system/event.h> -namespace NRainCheck { - class TTaskTracker; - - namespace NPrivate { - struct ITaskFactory { - virtual TIntrusivePtr<TTaskRunnerBase> NewTask(ISubtaskListener*) = 0; +namespace NRainCheck { + class TTaskTracker; + + namespace NPrivate { + struct ITaskFactory { + virtual TIntrusivePtr<TTaskRunnerBase> NewTask(ISubtaskListener*) = 0; virtual ~ITaskFactory() { } - }; - - struct TTaskTrackerReceipt: public ISubtaskListener, public TIntrusiveListItem<TTaskTrackerReceipt> { - TTaskTracker* const TaskTracker; - TIntrusivePtr<TTaskRunnerBase> Task; - + }; + + struct TTaskTrackerReceipt: public ISubtaskListener, public TIntrusiveListItem<TTaskTrackerReceipt> { + TTaskTracker* const TaskTracker; + TIntrusivePtr<TTaskRunnerBase> Task; + TTaskTrackerReceipt(TTaskTracker* taskTracker) : TaskTracker(taskTracker) { } - + void SetDone() override; - + TString GetStatusSingleLine(); - }; - - struct TTaskTrackerStatus { - ui32 Size; - }; - - } - - class TTaskTracker + }; + + struct TTaskTrackerStatus { + ui32 Size; + }; + + } + + class TTaskTracker : public TAtomicRefCount<TTaskTracker>, public NActor::TActor<TTaskTracker>, public NActor::TQueueInActor<TTaskTracker, NPrivate::ITaskFactory*>, public NActor::TQueueInActor<TTaskTracker, NPrivate::TTaskTrackerReceipt*>, public NActor::TQueueInActor<TTaskTracker, TAsyncResult<NPrivate::TTaskTrackerStatus>*> { - friend struct NPrivate::TTaskTrackerReceipt; - - private: - TAtomicBox<bool> ShutdownFlag; + friend struct NPrivate::TTaskTrackerReceipt; + + private: + TAtomicBox<bool> ShutdownFlag; TSystemEvent ShutdownEvent; - - TIntrusiveList<NPrivate::TTaskTrackerReceipt> Tasks; - - template <typename TItem> - NActor::TQueueInActor<TTaskTracker, TItem>* GetQueue() { - return this; - } - - public: - TTaskTracker(NActor::TExecutor* executor); + + TIntrusiveList<NPrivate::TTaskTrackerReceipt> Tasks; + + template <typename TItem> + NActor::TQueueInActor<TTaskTracker, TItem>* GetQueue() { + return this; + } + + public: + TTaskTracker(NActor::TExecutor* executor); ~TTaskTracker() override; - - void Shutdown(); - - void ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, NPrivate::ITaskFactory*); - void ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, NPrivate::TTaskTrackerReceipt*); - void ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, TAsyncResult<NPrivate::TTaskTrackerStatus>*); - - void Act(NActor::TDefaultTag); - - template <typename TTask, typename TEnv, typename TParam> - void Spawn(TEnv* env, TParam param) { - struct TTaskFactory: public NPrivate::ITaskFactory { - TEnv* const Env; - TParam Param; - + + void Shutdown(); + + void ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, NPrivate::ITaskFactory*); + void ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, NPrivate::TTaskTrackerReceipt*); + void ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, TAsyncResult<NPrivate::TTaskTrackerStatus>*); + + void Act(NActor::TDefaultTag); + + template <typename TTask, typename TEnv, typename TParam> + void Spawn(TEnv* env, TParam param) { + struct TTaskFactory: public NPrivate::ITaskFactory { + TEnv* const Env; + TParam Param; + TTaskFactory(TEnv* env, TParam param) : Env(env) , Param(param) { } - + TIntrusivePtr<TTaskRunnerBase> NewTask(ISubtaskListener* subtaskListener) override { - return NRainCheck::SpawnTask<TTask>(Env, Param, subtaskListener).Get(); - } - }; - - GetQueue<NPrivate::ITaskFactory*>()->EnqueueAndSchedule(new TTaskFactory(env, param)); - } - - ui32 Size(); - }; - -} + return NRainCheck::SpawnTask<TTask>(Env, Param, subtaskListener).Get(); + } + }; + + GetQueue<NPrivate::ITaskFactory*>()->EnqueueAndSchedule(new TTaskFactory(env, param)); + } + + ui32 Size(); + }; + +} diff --git a/library/cpp/messagebus/rain_check/core/track_ut.cpp b/library/cpp/messagebus/rain_check/core/track_ut.cpp index f2ac90fa3c..05f7de1319 100644 --- a/library/cpp/messagebus/rain_check/core/track_ut.cpp +++ b/library/cpp/messagebus/rain_check/core/track_ut.cpp @@ -1,45 +1,45 @@ #include <library/cpp/testing/unittest/registar.h> - + #include "track.h" #include <library/cpp/messagebus/rain_check/test/helper/misc.h> #include <library/cpp/messagebus/rain_check/test/ut/test.h> - -using namespace NRainCheck; - + +using namespace NRainCheck; + Y_UNIT_TEST_SUITE(TaskTracker) { - struct TTaskForTracker: public ISimpleTask { - TTestSync* const TestSync; - + struct TTaskForTracker: public ISimpleTask { + TTestSync* const TestSync; + TTaskForTracker(TTestEnv*, TTestSync* testSync) : TestSync(testSync) { } - + TContinueFunc Start() override { - TestSync->WaitForAndIncrement(0); - TestSync->WaitForAndIncrement(2); + TestSync->WaitForAndIncrement(0); + TestSync->WaitForAndIncrement(2); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(Simple) { - TTestEnv env; - - TIntrusivePtr<TTaskTracker> tracker(new TTaskTracker(env.GetExecutor())); - - TTestSync testSync; - - tracker->Spawn<TTaskForTracker>(&env, &testSync); - - testSync.WaitFor(1); - + TTestEnv env; + + TIntrusivePtr<TTaskTracker> tracker(new TTaskTracker(env.GetExecutor())); + + TTestSync testSync; + + tracker->Spawn<TTaskForTracker>(&env, &testSync); + + testSync.WaitFor(1); + UNIT_ASSERT_VALUES_EQUAL(1u, tracker->Size()); - - testSync.CheckAndIncrement(1); - - testSync.WaitForAndIncrement(3); - - tracker->Shutdown(); - } -} + + testSync.CheckAndIncrement(1); + + testSync.WaitForAndIncrement(3); + + tracker->Shutdown(); + } +} diff --git a/library/cpp/messagebus/rain_check/core/ya.make b/library/cpp/messagebus/rain_check/core/ya.make index 497e452729..c6fb5640d4 100644 --- a/library/cpp/messagebus/rain_check/core/ya.make +++ b/library/cpp/messagebus/rain_check/core/ya.make @@ -1,25 +1,25 @@ -LIBRARY() - +LIBRARY() + OWNER(g:messagebus) - -PEERDIR( + +PEERDIR( library/cpp/coroutine/engine library/cpp/deprecated/enum_codegen library/cpp/messagebus library/cpp/messagebus/actor library/cpp/messagebus/scheduler -) - -SRCS( - coro.cpp - coro_stack.cpp - env.cpp - rain_check.cpp - simple.cpp - sleep.cpp - spawn.cpp - task.cpp - track.cpp -) - -END() +) + +SRCS( + coro.cpp + coro_stack.cpp + env.cpp + rain_check.cpp + simple.cpp + sleep.cpp + spawn.cpp + task.cpp + track.cpp +) + +END() diff --git a/library/cpp/messagebus/rain_check/http/client_ut.cpp b/library/cpp/messagebus/rain_check/http/client_ut.cpp index c6e4a151bd..1628114391 100644 --- a/library/cpp/messagebus/rain_check/http/client_ut.cpp +++ b/library/cpp/messagebus/rain_check/http/client_ut.cpp @@ -25,7 +25,7 @@ #include <utility> using namespace NRainCheck; -using namespace NBus::NTest; +using namespace NBus::NTest; namespace { class THttpClientEnv: public TTestEnvTemplate<THttpClientEnv> { @@ -145,11 +145,11 @@ Y_UNIT_TEST_SUITE(RainCheckHttpClient) { static const TIpPort SERVER_PORT = 4000; Y_UNIT_TEST(Simple) { - // TODO: randomize port - if (!IsFixedPortTestAllowed()) { - return; - } - + // TODO: randomize port + if (!IsFixedPortTestAllowed()) { + return; + } + TSimpleServer server; NNeh::IServicesRef runner = RunServer(SERVER_PORT, server); diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_client.cpp b/library/cpp/messagebus/rain_check/messagebus/messagebus_client.cpp index 13d3132fb7..daac8d9a99 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_client.cpp +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_client.cpp @@ -1,98 +1,98 @@ -#include "messagebus_client.h" - -using namespace NRainCheck; -using namespace NBus; - -TBusClientService::TBusClientService( +#include "messagebus_client.h" + +using namespace NRainCheck; +using namespace NBus; + +TBusClientService::TBusClientService( const NBus::TBusSessionConfig& config, NBus::TBusProtocol* proto, NBus::TBusMessageQueue* queue) { - Session = queue->CreateSource(proto, this, config); -} - + Session = queue->CreateSource(proto, this, config); +} + TBusClientService::~TBusClientService() { - Session->Shutdown(); -} - + Session->Shutdown(); +} + void TBusClientService::SendCommon(NBus::TBusMessage* message, const NBus::TNetAddr&, TBusFuture* future) { - TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); - - future->SetRunning(current); - - future->Task = current; - - // after this statement message is owned by both messagebus and future - future->Request.Reset(message); - - // TODO: allow cookie in messagebus - message->Data = future; -} - -void TBusClientService::ProcessResultCommon(NBus::TBusMessageAutoPtr message, + TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); + + future->SetRunning(current); + + future->Task = current; + + // after this statement message is owned by both messagebus and future + future->Request.Reset(message); + + // TODO: allow cookie in messagebus + message->Data = future; +} + +void TBusClientService::ProcessResultCommon(NBus::TBusMessageAutoPtr message, const NBus::TNetAddr&, TBusFuture* future, NBus::EMessageStatus status) { Y_UNUSED(message.Release()); - - if (status == NBus::MESSAGE_OK) { - return; - } - + + if (status == NBus::MESSAGE_OK) { + return; + } + future->SetDoneAndSchedule(status, nullptr); -} - -void TBusClientService::SendOneWay( +} + +void TBusClientService::SendOneWay( NBus::TBusMessageAutoPtr message, const NBus::TNetAddr& addr, TBusFuture* future) { - SendCommon(message.Get(), addr, future); - - EMessageStatus ok = Session->SendMessageOneWay(message.Get(), &addr, false); - ProcessResultCommon(message, addr, future, ok); -} - + SendCommon(message.Get(), addr, future); + + EMessageStatus ok = Session->SendMessageOneWay(message.Get(), &addr, false); + ProcessResultCommon(message, addr, future, ok); +} + NBus::TBusClientSessionPtr TBusClientService::GetSessionForMonitoring() const { return Session; } -void TBusClientService::Send( +void TBusClientService::Send( TBusMessageAutoPtr message, const TNetAddr& addr, TBusFuture* future) { - SendCommon(message.Get(), addr, future); - - EMessageStatus ok = Session->SendMessage(message.Get(), &addr, false); - ProcessResultCommon(message, addr, future, ok); -} - -void TBusClientService::OnReply( + SendCommon(message.Get(), addr, future); + + EMessageStatus ok = Session->SendMessage(message.Get(), &addr, false); + ProcessResultCommon(message, addr, future, ok); +} + +void TBusClientService::OnReply( TAutoPtr<TBusMessage> request, TAutoPtr<TBusMessage> response) { TBusFuture* future = (TBusFuture*)request->Data; Y_ASSERT(future->Request.Get() == request.Get()); Y_UNUSED(request.Release()); - future->SetDoneAndSchedule(MESSAGE_OK, response); -} - -void NRainCheck::TBusClientService::OnMessageSentOneWay( + future->SetDoneAndSchedule(MESSAGE_OK, response); +} + +void NRainCheck::TBusClientService::OnMessageSentOneWay( TAutoPtr<NBus::TBusMessage> request) { TBusFuture* future = (TBusFuture*)request->Data; Y_ASSERT(future->Request.Get() == request.Get()); Y_UNUSED(request.Release()); future->SetDoneAndSchedule(MESSAGE_OK, nullptr); -} - -void TBusClientService::OnError( +} + +void TBusClientService::OnError( TAutoPtr<TBusMessage> message, NBus::EMessageStatus status) { if (message->Data == nullptr) { - return; - } - + return; + } + TBusFuture* future = (TBusFuture*)message->Data; Y_ASSERT(future->Request.Get() == message.Get()); Y_UNUSED(message.Release()); future->SetDoneAndSchedule(status, nullptr); -} - +} + void TBusFuture::SetDoneAndSchedule(EMessageStatus status, TAutoPtr<TBusMessage> response) { - Status = status; - Response.Reset(response.Release()); - SetDone(); -} + Status = status; + Response.Reset(response.Release()); + SetDone(); +} diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_client.h b/library/cpp/messagebus/rain_check/messagebus/messagebus_client.h index 8bcc03b8d9..0a291cdea6 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_client.h +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_client.h @@ -1,67 +1,67 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/rain_check/core/task.h> #include <library/cpp/messagebus/ybus.h> - -namespace NRainCheck { - class TBusFuture: public TSubtaskCompletion { - friend class TBusClientService; - - private: - THolder<NBus::TBusMessage> Request; - THolder<NBus::TBusMessage> Response; - NBus::EMessageStatus Status; - - private: - TTaskRunnerBase* Task; - - void SetDoneAndSchedule(NBus::EMessageStatus, TAutoPtr<NBus::TBusMessage>); - - public: - // TODO: add MESSAGE_UNDEFINED + +namespace NRainCheck { + class TBusFuture: public TSubtaskCompletion { + friend class TBusClientService; + + private: + THolder<NBus::TBusMessage> Request; + THolder<NBus::TBusMessage> Response; + NBus::EMessageStatus Status; + + private: + TTaskRunnerBase* Task; + + void SetDoneAndSchedule(NBus::EMessageStatus, TAutoPtr<NBus::TBusMessage>); + + public: + // TODO: add MESSAGE_UNDEFINED TBusFuture() : Status(NBus::MESSAGE_DONT_ASK) , Task(nullptr) { } - - NBus::TBusMessage* GetRequest() const { - return Request.Get(); - } - - NBus::TBusMessage* GetResponse() const { + + NBus::TBusMessage* GetRequest() const { + return Request.Get(); + } + + NBus::TBusMessage* GetResponse() const { Y_ASSERT(IsDone()); - return Response.Get(); - } - - NBus::EMessageStatus GetStatus() const { + return Response.Get(); + } + + NBus::EMessageStatus GetStatus() const { Y_ASSERT(IsDone()); - return Status; - } - }; - - class TBusClientService: private NBus::IBusClientHandler { - private: - NBus::TBusClientSessionPtr Session; - - public: - TBusClientService(const NBus::TBusSessionConfig&, NBus::TBusProtocol*, NBus::TBusMessageQueue*); + return Status; + } + }; + + class TBusClientService: private NBus::IBusClientHandler { + private: + NBus::TBusClientSessionPtr Session; + + public: + TBusClientService(const NBus::TBusSessionConfig&, NBus::TBusProtocol*, NBus::TBusMessageQueue*); ~TBusClientService() override; - - void Send(NBus::TBusMessageAutoPtr, const NBus::TNetAddr&, TBusFuture* future); - void SendOneWay(NBus::TBusMessageAutoPtr, const NBus::TNetAddr&, TBusFuture* future); - + + void Send(NBus::TBusMessageAutoPtr, const NBus::TNetAddr&, TBusFuture* future); + void SendOneWay(NBus::TBusMessageAutoPtr, const NBus::TNetAddr&, TBusFuture* future); + // Use it only for monitoring NBus::TBusClientSessionPtr GetSessionForMonitoring() const; - private: - void SendCommon(NBus::TBusMessage*, const NBus::TNetAddr&, TBusFuture* future); - void ProcessResultCommon(NBus::TBusMessageAutoPtr, const NBus::TNetAddr&, TBusFuture* future, NBus::EMessageStatus); - + private: + void SendCommon(NBus::TBusMessage*, const NBus::TNetAddr&, TBusFuture* future); + void ProcessResultCommon(NBus::TBusMessageAutoPtr, const NBus::TNetAddr&, TBusFuture* future, NBus::EMessageStatus); + void OnReply(TAutoPtr<NBus::TBusMessage> pMessage, TAutoPtr<NBus::TBusMessage> pReply) override; void OnError(TAutoPtr<NBus::TBusMessage> pMessage, NBus::EMessageStatus status) override; void OnMessageSentOneWay(TAutoPtr<NBus::TBusMessage>) override; - }; - -} + }; + +} diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_client_ut.cpp b/library/cpp/messagebus/rain_check/messagebus/messagebus_client_ut.cpp index 4571f6f74a..1b3618558b 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_client_ut.cpp +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_client_ut.cpp @@ -1,146 +1,146 @@ #include <library/cpp/testing/unittest/registar.h> - + #include "messagebus_client.h" #include <library/cpp/messagebus/rain_check/test/ut/test.h> #include <library/cpp/messagebus/test/helper/example.h> #include <library/cpp/messagebus/test/helper/object_count_check.h> - + #include <util/generic/cast.h> - -using namespace NBus; -using namespace NBus::NTest; -using namespace NRainCheck; - -struct TMessageBusClientEnv: public TTestEnvTemplate<TMessageBusClientEnv> { - // TODO: use same thread pool - TBusMessageQueuePtr Queue; - TExampleProtocol Proto; - TBusClientService BusClientService; - - static TBusQueueConfig QueueConfig() { - TBusQueueConfig r; - r.NumWorkers = 4; - return r; - } - - TMessageBusClientEnv() - : Queue(CreateMessageQueue(GetExecutor())) - , BusClientService(TBusSessionConfig(), &Proto, Queue.Get()) + +using namespace NBus; +using namespace NBus::NTest; +using namespace NRainCheck; + +struct TMessageBusClientEnv: public TTestEnvTemplate<TMessageBusClientEnv> { + // TODO: use same thread pool + TBusMessageQueuePtr Queue; + TExampleProtocol Proto; + TBusClientService BusClientService; + + static TBusQueueConfig QueueConfig() { + TBusQueueConfig r; + r.NumWorkers = 4; + return r; + } + + TMessageBusClientEnv() + : Queue(CreateMessageQueue(GetExecutor())) + , BusClientService(TBusSessionConfig(), &Proto, Queue.Get()) { } -}; - +}; + Y_UNIT_TEST_SUITE(RainCheckMessageBusClient) { - struct TSimpleTask: public ISimpleTask { - TMessageBusClientEnv* const Env; - - const unsigned ServerPort; - - TSimpleTask(TMessageBusClientEnv* env, unsigned serverPort) - : Env(env) - , ServerPort(serverPort) - { - } - + struct TSimpleTask: public ISimpleTask { + TMessageBusClientEnv* const Env; + + const unsigned ServerPort; + + TSimpleTask(TMessageBusClientEnv* env, unsigned serverPort) + : Env(env) + , ServerPort(serverPort) + { + } + TVector<TSimpleSharedPtr<TBusFuture>> Requests; - + TContinueFunc Start() override { - for (unsigned i = 0; i < 3; ++i) { - Requests.push_back(new TBusFuture); - TNetAddr addr("localhost", ServerPort); - Env->BusClientService.Send(new TExampleRequest(&Env->Proto.RequestCount), addr, Requests[i].Get()); - } - - return TContinueFunc(&TSimpleTask::GotReplies); - } - - TContinueFunc GotReplies() { - for (unsigned i = 0; i < Requests.size(); ++i) { + for (unsigned i = 0; i < 3; ++i) { + Requests.push_back(new TBusFuture); + TNetAddr addr("localhost", ServerPort); + Env->BusClientService.Send(new TExampleRequest(&Env->Proto.RequestCount), addr, Requests[i].Get()); + } + + return TContinueFunc(&TSimpleTask::GotReplies); + } + + TContinueFunc GotReplies() { + for (unsigned i = 0; i < Requests.size(); ++i) { Y_VERIFY(Requests[i]->GetStatus() == MESSAGE_OK); - VerifyDynamicCast<TExampleResponse*>(Requests[i]->GetResponse()); - } - Env->TestSync.CheckAndIncrement(0); + VerifyDynamicCast<TExampleResponse*>(Requests[i]->GetResponse()); + } + Env->TestSync.CheckAndIncrement(0); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(Simple) { - TObjectCountCheck objectCountCheck; - - TExampleServer server; - - TMessageBusClientEnv env; - - TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TSimpleTask>(server.GetActualListenPort()); - - env.TestSync.WaitForAndIncrement(1); - } - - struct TOneWayServer: public NBus::IBusServerHandler { - TTestSync* const TestSync; - TExampleProtocol Proto; - NBus::TBusMessageQueuePtr Queue; - NBus::TBusServerSessionPtr Session; - - TOneWayServer(TTestSync* testSync) - : TestSync(testSync) - { - Queue = CreateMessageQueue(); - Session = Queue->CreateDestination(&Proto, this, NBus::TBusSessionConfig()); - } - + TObjectCountCheck objectCountCheck; + + TExampleServer server; + + TMessageBusClientEnv env; + + TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TSimpleTask>(server.GetActualListenPort()); + + env.TestSync.WaitForAndIncrement(1); + } + + struct TOneWayServer: public NBus::IBusServerHandler { + TTestSync* const TestSync; + TExampleProtocol Proto; + NBus::TBusMessageQueuePtr Queue; + NBus::TBusServerSessionPtr Session; + + TOneWayServer(TTestSync* testSync) + : TestSync(testSync) + { + Queue = CreateMessageQueue(); + Session = Queue->CreateDestination(&Proto, this, NBus::TBusSessionConfig()); + } + void OnMessage(NBus::TOnMessageContext& context) override { - TestSync->CheckAndIncrement(1); - context.ForgetRequest(); - } - }; - - struct TOneWayTask: public ISimpleTask { - TMessageBusClientEnv* const Env; - - const unsigned ServerPort; - - TOneWayTask(TMessageBusClientEnv* env, unsigned serverPort) - : Env(env) - , ServerPort(serverPort) - { - } - + TestSync->CheckAndIncrement(1); + context.ForgetRequest(); + } + }; + + struct TOneWayTask: public ISimpleTask { + TMessageBusClientEnv* const Env; + + const unsigned ServerPort; + + TOneWayTask(TMessageBusClientEnv* env, unsigned serverPort) + : Env(env) + , ServerPort(serverPort) + { + } + TVector<TSimpleSharedPtr<TBusFuture>> Requests; - + TContinueFunc Start() override { - Env->TestSync.CheckAndIncrement(0); - - for (unsigned i = 0; i < 1; ++i) { - Requests.push_back(new TBusFuture); - TNetAddr addr("localhost", ServerPort); - Env->BusClientService.SendOneWay(new TExampleRequest(&Env->Proto.RequestCount), addr, Requests[i].Get()); - } - - return TContinueFunc(&TOneWayTask::GotReplies); - } - - TContinueFunc GotReplies() { - for (unsigned i = 0; i < Requests.size(); ++i) { + Env->TestSync.CheckAndIncrement(0); + + for (unsigned i = 0; i < 1; ++i) { + Requests.push_back(new TBusFuture); + TNetAddr addr("localhost", ServerPort); + Env->BusClientService.SendOneWay(new TExampleRequest(&Env->Proto.RequestCount), addr, Requests[i].Get()); + } + + return TContinueFunc(&TOneWayTask::GotReplies); + } + + TContinueFunc GotReplies() { + for (unsigned i = 0; i < Requests.size(); ++i) { Y_VERIFY(Requests[i]->GetStatus() == MESSAGE_OK); Y_VERIFY(!Requests[i]->GetResponse()); - } - Env->TestSync.WaitForAndIncrement(2); + } + Env->TestSync.WaitForAndIncrement(2); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(OneWay) { - TObjectCountCheck objectCountCheck; - - TMessageBusClientEnv env; - - TOneWayServer server(&env.TestSync); - - TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TOneWayTask>(server.Session->GetActualListenPort()); - - env.TestSync.WaitForAndIncrement(3); - } -} + TObjectCountCheck objectCountCheck; + + TMessageBusClientEnv env; + + TOneWayServer server(&env.TestSync); + + TIntrusivePtr<TSimpleTaskRunner> task = env.SpawnTask<TOneWayTask>(server.Session->GetActualListenPort()); + + env.TestSync.WaitForAndIncrement(3); + } +} diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_server.cpp b/library/cpp/messagebus/rain_check/messagebus/messagebus_server.cpp index 1346ef3243..5d4b13d664 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_server.cpp +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_server.cpp @@ -1,17 +1,17 @@ #include "messagebus_server.h" #include <library/cpp/messagebus/rain_check/core/spawn.h> - -using namespace NRainCheck; - -TBusTaskStarter::TBusTaskStarter(TAutoPtr<ITaskFactory> taskFactory) - : TaskFactory(taskFactory) -{ -} - + +using namespace NRainCheck; + +TBusTaskStarter::TBusTaskStarter(TAutoPtr<ITaskFactory> taskFactory) + : TaskFactory(taskFactory) +{ +} + void TBusTaskStarter::OnMessage(NBus::TOnMessageContext& onMessage) { - TaskFactory->NewTask(onMessage); -} - + TaskFactory->NewTask(onMessage); +} + TBusTaskStarter::~TBusTaskStarter() { -} +} diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_server.h b/library/cpp/messagebus/rain_check/messagebus/messagebus_server.h index 28d016599a..1334f05fe4 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_server.h +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_server.h @@ -1,46 +1,46 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/rain_check/core/spawn.h> #include <library/cpp/messagebus/rain_check/core/task.h> - + #include <library/cpp/messagebus/ybus.h> - + #include <util/system/yassert.h> - -namespace NRainCheck { - class TBusTaskStarter: public NBus::IBusServerHandler { - private: - struct ITaskFactory { - virtual void NewTask(NBus::TOnMessageContext&) = 0; + +namespace NRainCheck { + class TBusTaskStarter: public NBus::IBusServerHandler { + private: + struct ITaskFactory { + virtual void NewTask(NBus::TOnMessageContext&) = 0; virtual ~ITaskFactory() { } - }; - - THolder<ITaskFactory> TaskFactory; - + }; + + THolder<ITaskFactory> TaskFactory; + void OnMessage(NBus::TOnMessageContext&) override; - public: - TBusTaskStarter(TAutoPtr<ITaskFactory>); + public: + TBusTaskStarter(TAutoPtr<ITaskFactory>); ~TBusTaskStarter() override; - - public: - template <typename TTask, typename TEnv> - static TAutoPtr<TBusTaskStarter> NewStarter(TEnv* env) { - struct TTaskFactory: public ITaskFactory { - TEnv* const Env; - + + public: + template <typename TTask, typename TEnv> + static TAutoPtr<TBusTaskStarter> NewStarter(TEnv* env) { + struct TTaskFactory: public ITaskFactory { + TEnv* const Env; + TTaskFactory(TEnv* env) : Env(env) { } - + void NewTask(NBus::TOnMessageContext& context) override { - SpawnTask<TTask, TEnv, NBus::TOnMessageContext&>(Env, context); - } - }; - - return new TBusTaskStarter(new TTaskFactory(env)); - } - }; -} + SpawnTask<TTask, TEnv, NBus::TOnMessageContext&>(Env, context); + } + }; + + return new TBusTaskStarter(new TTaskFactory(env)); + } + }; +} diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_server_ut.cpp b/library/cpp/messagebus/rain_check/messagebus/messagebus_server_ut.cpp index fcb718c3ba..7c11399f1b 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_server_ut.cpp +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_server_ut.cpp @@ -1,51 +1,51 @@ #include <library/cpp/testing/unittest/registar.h> - + #include "messagebus_server.h" - + #include <library/cpp/messagebus/rain_check/test/ut/test.h> - + #include <library/cpp/messagebus/test/helper/example.h> - -using namespace NBus; -using namespace NBus::NTest; -using namespace NRainCheck; - -struct TMessageBusServerEnv: public TTestEnvTemplate<TMessageBusServerEnv> { - TExampleProtocol Proto; -}; - + +using namespace NBus; +using namespace NBus::NTest; +using namespace NRainCheck; + +struct TMessageBusServerEnv: public TTestEnvTemplate<TMessageBusServerEnv> { + TExampleProtocol Proto; +}; + Y_UNIT_TEST_SUITE(RainCheckMessageBusServer) { - struct TSimpleServerTask: public ISimpleTask { - private: - TMessageBusServerEnv* const Env; - TOnMessageContext MessageContext; - - public: - TSimpleServerTask(TMessageBusServerEnv* env, TOnMessageContext& messageContext) - : Env(env) - { - MessageContext.Swap(messageContext); - } - + struct TSimpleServerTask: public ISimpleTask { + private: + TMessageBusServerEnv* const Env; + TOnMessageContext MessageContext; + + public: + TSimpleServerTask(TMessageBusServerEnv* env, TOnMessageContext& messageContext) + : Env(env) + { + MessageContext.Swap(messageContext); + } + TContinueFunc Start() override { - MessageContext.SendReplyMove(new TExampleResponse(&Env->Proto.ResponseCount)); + MessageContext.SendReplyMove(new TExampleResponse(&Env->Proto.ResponseCount)); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(Simple) { - TMessageBusServerEnv env; - - THolder<TBusTaskStarter> starter(TBusTaskStarter::NewStarter<TSimpleServerTask>(&env)); - - TBusMessageQueuePtr queue(CreateMessageQueue(env.GetExecutor())); - - TExampleProtocol proto; - - TBusServerSessionPtr session = queue->CreateDestination(&env.Proto, starter.Get(), TBusSessionConfig()); - - TExampleClient client; - - client.SendMessagesWaitReplies(1, TNetAddr("localhost", session->GetActualListenPort())); - } -} + TMessageBusServerEnv env; + + THolder<TBusTaskStarter> starter(TBusTaskStarter::NewStarter<TSimpleServerTask>(&env)); + + TBusMessageQueuePtr queue(CreateMessageQueue(env.GetExecutor())); + + TExampleProtocol proto; + + TBusServerSessionPtr session = queue->CreateDestination(&env.Proto, starter.Get(), TBusSessionConfig()); + + TExampleClient client; + + client.SendMessagesWaitReplies(1, TNetAddr("localhost", session->GetActualListenPort())); + } +} diff --git a/library/cpp/messagebus/rain_check/messagebus/ya.make b/library/cpp/messagebus/rain_check/messagebus/ya.make index d7dc902ad1..defdac9a61 100644 --- a/library/cpp/messagebus/rain_check/messagebus/ya.make +++ b/library/cpp/messagebus/rain_check/messagebus/ya.make @@ -1,15 +1,15 @@ -LIBRARY() - +LIBRARY() + OWNER(g:messagebus) - -PEERDIR( + +PEERDIR( library/cpp/messagebus library/cpp/messagebus/rain_check/core -) - -SRCS( - messagebus_client.cpp - messagebus_server.cpp -) - -END() +) + +SRCS( + messagebus_client.cpp + messagebus_server.cpp +) + +END() diff --git a/library/cpp/messagebus/rain_check/test/helper/misc.cpp b/library/cpp/messagebus/rain_check/test/helper/misc.cpp index 2a75c42744..c0fcb27252 100644 --- a/library/cpp/messagebus/rain_check/test/helper/misc.cpp +++ b/library/cpp/messagebus/rain_check/test/helper/misc.cpp @@ -1,27 +1,27 @@ #include "misc.h" -#include <util/system/yassert.h> - -using namespace NRainCheck; - +#include <util/system/yassert.h> + +using namespace NRainCheck; + void TSpawnNopTasksCoroTask::Run() { Y_VERIFY(Count <= Completion.size()); - for (unsigned i = 0; i < Count; ++i) { - SpawnSubtask<TNopCoroTask>(Env, &Completion[i], ""); - } - - WaitForSubtasks(); -} - + for (unsigned i = 0; i < Count; ++i) { + SpawnSubtask<TNopCoroTask>(Env, &Completion[i], ""); + } + + WaitForSubtasks(); +} + TContinueFunc TSpawnNopTasksSimpleTask::Start() { Y_VERIFY(Count <= Completion.size()); - for (unsigned i = 0; i < Count; ++i) { - SpawnSubtask<TNopSimpleTask>(Env, &Completion[i], ""); - } - - return &TSpawnNopTasksSimpleTask::Join; -} - + for (unsigned i = 0; i < Count; ++i) { + SpawnSubtask<TNopSimpleTask>(Env, &Completion[i], ""); + } + + return &TSpawnNopTasksSimpleTask::Join; +} + TContinueFunc TSpawnNopTasksSimpleTask::Join() { return nullptr; -} +} diff --git a/library/cpp/messagebus/rain_check/test/helper/misc.h b/library/cpp/messagebus/rain_check/test/helper/misc.h index dbcc04778d..9150be4d2f 100644 --- a/library/cpp/messagebus/rain_check/test/helper/misc.h +++ b/library/cpp/messagebus/rain_check/test/helper/misc.h @@ -1,57 +1,57 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/rain_check/core/rain_check.h> - + #include <array> - -namespace NRainCheck { - struct TNopSimpleTask: public ISimpleTask { + +namespace NRainCheck { + struct TNopSimpleTask: public ISimpleTask { TNopSimpleTask(IEnv*, const void*) { } - + TContinueFunc Start() override { return nullptr; - } - }; - - struct TNopCoroTask: public ICoroTask { + } + }; + + struct TNopCoroTask: public ICoroTask { TNopCoroTask(IEnv*, const void*) { } - + void Run() override { } - }; - - struct TSpawnNopTasksCoroTask: public ICoroTask { - IEnv* const Env; - unsigned const Count; - - TSpawnNopTasksCoroTask(IEnv* env, unsigned count) - : Env(env) - , Count(count) + }; + + struct TSpawnNopTasksCoroTask: public ICoroTask { + IEnv* const Env; + unsigned const Count; + + TSpawnNopTasksCoroTask(IEnv* env, unsigned count) + : Env(env) + , Count(count) { } - + std::array<TSubtaskCompletion, 2> Completion; - + void Run() override; - }; - - struct TSpawnNopTasksSimpleTask: public ISimpleTask { - IEnv* const Env; - unsigned const Count; - - TSpawnNopTasksSimpleTask(IEnv* env, unsigned count) - : Env(env) - , Count(count) + }; + + struct TSpawnNopTasksSimpleTask: public ISimpleTask { + IEnv* const Env; + unsigned const Count; + + TSpawnNopTasksSimpleTask(IEnv* env, unsigned count) + : Env(env) + , Count(count) { } - + std::array<TSubtaskCompletion, 2> Completion; - + TContinueFunc Start() override; - - TContinueFunc Join(); - }; - -} + + TContinueFunc Join(); + }; + +} diff --git a/library/cpp/messagebus/rain_check/test/helper/ya.make b/library/cpp/messagebus/rain_check/test/helper/ya.make index 08265167a7..aa9e4e6d81 100644 --- a/library/cpp/messagebus/rain_check/test/helper/ya.make +++ b/library/cpp/messagebus/rain_check/test/helper/ya.make @@ -1,13 +1,13 @@ -LIBRARY(messagebus-rain_check-test-helper) - +LIBRARY(messagebus-rain_check-test-helper) + OWNER(g:messagebus) - + PEERDIR( library/cpp/messagebus/rain_check/core ) -SRCS( - misc.cpp -) - -END() +SRCS( + misc.cpp +) + +END() diff --git a/library/cpp/messagebus/rain_check/test/perftest/perftest.cpp b/library/cpp/messagebus/rain_check/test/perftest/perftest.cpp index d0c6451f47..22edbd8c6b 100644 --- a/library/cpp/messagebus/rain_check/test/perftest/perftest.cpp +++ b/library/cpp/messagebus/rain_check/test/perftest/perftest.cpp @@ -1,154 +1,154 @@ #include <library/cpp/messagebus/rain_check/test/helper/misc.h> - + #include <library/cpp/messagebus/rain_check/core/rain_check.h> - + #include <util/datetime/base.h> #include <array> - -using namespace NRainCheck; - -static const unsigned SUBTASKS = 2; - -struct TRainCheckPerftestEnv: public TSimpleEnvTemplate<TRainCheckPerftestEnv> { - unsigned SubtasksPerTask; - - TRainCheckPerftestEnv() - : TSimpleEnvTemplate<TRainCheckPerftestEnv>(4) - , SubtasksPerTask(1000) + +using namespace NRainCheck; + +static const unsigned SUBTASKS = 2; + +struct TRainCheckPerftestEnv: public TSimpleEnvTemplate<TRainCheckPerftestEnv> { + unsigned SubtasksPerTask; + + TRainCheckPerftestEnv() + : TSimpleEnvTemplate<TRainCheckPerftestEnv>(4) + , SubtasksPerTask(1000) { } -}; - -struct TCoroOuter: public ICoroTask { - TRainCheckPerftestEnv* const Env; - +}; + +struct TCoroOuter: public ICoroTask { + TRainCheckPerftestEnv* const Env; + TCoroOuter(TRainCheckPerftestEnv* env) : Env(env) { } - + void Run() override { - for (;;) { - TInstant start = TInstant::Now(); - - unsigned count = 0; - - unsigned current = 1000; - - do { - for (unsigned i = 0; i < current; ++i) { + for (;;) { + TInstant start = TInstant::Now(); + + unsigned count = 0; + + unsigned current = 1000; + + do { + for (unsigned i = 0; i < current; ++i) { std::array<TSubtaskCompletion, SUBTASKS> completion; - - for (unsigned j = 0; j < SUBTASKS; ++j) { - //SpawnSubtask<TNopSimpleTask>(Env, &completion[j]); - //SpawnSubtask<TSpawnNopTasksCoroTask>(Env, &completion[j], SUBTASKS); - SpawnSubtask<TSpawnNopTasksSimpleTask>(Env, &completion[j], SUBTASKS); - } - - WaitForSubtasks(); - } - - count += current; - current *= 2; - } while (TInstant::Now() - start < TDuration::Seconds(1)); - - TDuration d = TInstant::Now() - start; - unsigned dns = d.NanoSeconds() / count; - Cerr << dns << "ns per spawn/join\n"; - } - } -}; - -struct TSimpleOuter: public ISimpleTask { - TRainCheckPerftestEnv* const Env; - + + for (unsigned j = 0; j < SUBTASKS; ++j) { + //SpawnSubtask<TNopSimpleTask>(Env, &completion[j]); + //SpawnSubtask<TSpawnNopTasksCoroTask>(Env, &completion[j], SUBTASKS); + SpawnSubtask<TSpawnNopTasksSimpleTask>(Env, &completion[j], SUBTASKS); + } + + WaitForSubtasks(); + } + + count += current; + current *= 2; + } while (TInstant::Now() - start < TDuration::Seconds(1)); + + TDuration d = TInstant::Now() - start; + unsigned dns = d.NanoSeconds() / count; + Cerr << dns << "ns per spawn/join\n"; + } + } +}; + +struct TSimpleOuter: public ISimpleTask { + TRainCheckPerftestEnv* const Env; + TSimpleOuter(TRainCheckPerftestEnv* env, const void*) : Env(env) { } - - TInstant StartInstant; - unsigned Count; - unsigned Current; - unsigned I; - + + TInstant StartInstant; + unsigned Count; + unsigned Current; + unsigned I; + TContinueFunc Start() override { - StartInstant = TInstant::Now(); - Count = 0; - Current = 1000; - I = 0; - - return &TSimpleOuter::Spawn; - } - + StartInstant = TInstant::Now(); + Count = 0; + Current = 1000; + I = 0; + + return &TSimpleOuter::Spawn; + } + std::array<TSubtaskCompletion, SUBTASKS> Completion; - - TContinueFunc Spawn() { - for (unsigned j = 0; j < SUBTASKS; ++j) { - //SpawnSubtask<TNopSimpleTask>(Env, &Completion[j]); - //SpawnSubtask<TSpawnNopTasksCoroTask>(Env, &Completion[j], SUBTASKS); - SpawnSubtask<TSpawnNopTasksSimpleTask>(Env, &Completion[j], SUBTASKS); - } - - return &TSimpleOuter::Join; - } - - TContinueFunc Join() { - I += 1; - if (I != Current) { - return &TSimpleOuter::Spawn; - } - - I = 0; - Count += Current; - Current *= 2; - - TDuration d = TInstant::Now() - StartInstant; - if (d < TDuration::Seconds(1)) { - return &TSimpleOuter::Spawn; - } - - unsigned dns = d.NanoSeconds() / Count; - Cerr << dns << "ns per spawn/join\n"; - - return &TSimpleOuter::Start; - } -}; - -struct TReproduceCrashTask: public ISimpleTask { - TRainCheckPerftestEnv* const Env; - + + TContinueFunc Spawn() { + for (unsigned j = 0; j < SUBTASKS; ++j) { + //SpawnSubtask<TNopSimpleTask>(Env, &Completion[j]); + //SpawnSubtask<TSpawnNopTasksCoroTask>(Env, &Completion[j], SUBTASKS); + SpawnSubtask<TSpawnNopTasksSimpleTask>(Env, &Completion[j], SUBTASKS); + } + + return &TSimpleOuter::Join; + } + + TContinueFunc Join() { + I += 1; + if (I != Current) { + return &TSimpleOuter::Spawn; + } + + I = 0; + Count += Current; + Current *= 2; + + TDuration d = TInstant::Now() - StartInstant; + if (d < TDuration::Seconds(1)) { + return &TSimpleOuter::Spawn; + } + + unsigned dns = d.NanoSeconds() / Count; + Cerr << dns << "ns per spawn/join\n"; + + return &TSimpleOuter::Start; + } +}; + +struct TReproduceCrashTask: public ISimpleTask { + TRainCheckPerftestEnv* const Env; + TReproduceCrashTask(TRainCheckPerftestEnv* env) : Env(env) { } - + std::array<TSubtaskCompletion, SUBTASKS> Completion; - + TContinueFunc Start() override { - for (unsigned j = 0; j < 2; ++j) { - //SpawnSubtask<TNopSimpleTask>(Env, &Completion[j]); - SpawnSubtask<TSpawnNopTasksSimpleTask>(Env, &Completion[j], SUBTASKS); - } - - return &TReproduceCrashTask::Start; - } -}; - -int main(int argc, char** argv) { + for (unsigned j = 0; j < 2; ++j) { + //SpawnSubtask<TNopSimpleTask>(Env, &Completion[j]); + SpawnSubtask<TSpawnNopTasksSimpleTask>(Env, &Completion[j], SUBTASKS); + } + + return &TReproduceCrashTask::Start; + } +}; + +int main(int argc, char** argv) { Y_UNUSED(argc); Y_UNUSED(argv); - - TRainCheckPerftestEnv env; - - env.SpawnTask<TSimpleOuter>(""); - //env.SpawnTask<TCoroOuter>(); - //env.SpawnTask<TReproduceCrashTask>(); - - for (;;) { - Sleep(TDuration::Hours(1)); - } - - return 0; -} + + TRainCheckPerftestEnv env; + + env.SpawnTask<TSimpleOuter>(""); + //env.SpawnTask<TCoroOuter>(); + //env.SpawnTask<TReproduceCrashTask>(); + + for (;;) { + Sleep(TDuration::Hours(1)); + } + + return 0; +} diff --git a/library/cpp/messagebus/rain_check/test/perftest/ya.make b/library/cpp/messagebus/rain_check/test/perftest/ya.make index f80ddf2c05..7330a71700 100644 --- a/library/cpp/messagebus/rain_check/test/perftest/ya.make +++ b/library/cpp/messagebus/rain_check/test/perftest/ya.make @@ -1,14 +1,14 @@ -PROGRAM(messagebus_rain_check_perftest) - +PROGRAM(messagebus_rain_check_perftest) + OWNER(g:messagebus) - -PEERDIR( + +PEERDIR( library/cpp/messagebus/rain_check/core library/cpp/messagebus/rain_check/test/helper -) - -SRCS( - perftest.cpp -) - -END() +) + +SRCS( + perftest.cpp +) + +END() diff --git a/library/cpp/messagebus/rain_check/test/ut/test.h b/library/cpp/messagebus/rain_check/test/ut/test.h index 922f0f06cb..724f6b7530 100644 --- a/library/cpp/messagebus/rain_check/test/ut/test.h +++ b/library/cpp/messagebus/rain_check/test/ut/test.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/rain_check/core/rain_check.h> #include <library/cpp/messagebus/misc/test_sync.h> - -template <typename TSelf> -struct TTestEnvTemplate: public NRainCheck::TSimpleEnvTemplate<TSelf> { - TTestSync TestSync; -}; - -struct TTestEnv: public TTestEnvTemplate<TTestEnv> { -}; + +template <typename TSelf> +struct TTestEnvTemplate: public NRainCheck::TSimpleEnvTemplate<TSelf> { + TTestSync TestSync; +}; + +struct TTestEnv: public TTestEnvTemplate<TTestEnv> { +}; diff --git a/library/cpp/messagebus/rain_check/test/ut/ya.make b/library/cpp/messagebus/rain_check/test/ut/ya.make index 6191fe9fe0..9f7a93417a 100644 --- a/library/cpp/messagebus/rain_check/test/ut/ya.make +++ b/library/cpp/messagebus/rain_check/test/ut/ya.make @@ -1,24 +1,24 @@ PROGRAM(library-messagebus-rain_check-test-ut) - + OWNER(g:messagebus) - -PEERDIR( + +PEERDIR( library/cpp/testing/unittest_main library/cpp/messagebus/rain_check/core library/cpp/messagebus/rain_check/http library/cpp/messagebus/rain_check/messagebus library/cpp/messagebus/test/helper -) - -SRCS( - ../../core/coro_ut.cpp - ../../core/simple_ut.cpp - ../../core/sleep_ut.cpp - ../../core/spawn_ut.cpp - ../../core/track_ut.cpp +) + +SRCS( + ../../core/coro_ut.cpp + ../../core/simple_ut.cpp + ../../core/sleep_ut.cpp + ../../core/spawn_ut.cpp + ../../core/track_ut.cpp ../../http/client_ut.cpp - ../../messagebus/messagebus_client_ut.cpp - ../../messagebus/messagebus_server_ut.cpp -) - -END() + ../../messagebus/messagebus_client_ut.cpp + ../../messagebus/messagebus_server_ut.cpp +) + +END() diff --git a/library/cpp/messagebus/rain_check/test/ya.make b/library/cpp/messagebus/rain_check/test/ya.make index 83cdb16977..4c1d6f8161 100644 --- a/library/cpp/messagebus/rain_check/test/ya.make +++ b/library/cpp/messagebus/rain_check/test/ya.make @@ -1,6 +1,6 @@ OWNER(g:messagebus) -RECURSE( +RECURSE( perftest ut -) +) diff --git a/library/cpp/messagebus/rain_check/ya.make b/library/cpp/messagebus/rain_check/ya.make index c408615f42..966d54c232 100644 --- a/library/cpp/messagebus/rain_check/ya.make +++ b/library/cpp/messagebus/rain_check/ya.make @@ -1,8 +1,8 @@ OWNER(g:messagebus) -RECURSE( +RECURSE( core http messagebus test -) +) diff --git a/library/cpp/messagebus/ref_counted.h b/library/cpp/messagebus/ref_counted.h index bbe908b7c2..29b87764e3 100644 --- a/library/cpp/messagebus/ref_counted.h +++ b/library/cpp/messagebus/ref_counted.h @@ -1,6 +1,6 @@ -#pragma once - +#pragma once + class TAtomicRefCountedObject: public TAtomicRefCount<TAtomicRefCountedObject> { virtual ~TAtomicRefCountedObject() { } -}; +}; diff --git a/library/cpp/messagebus/remote_client_connection.cpp b/library/cpp/messagebus/remote_client_connection.cpp index b7b05e7bed..8c7a6db3a8 100644 --- a/library/cpp/messagebus/remote_client_connection.cpp +++ b/library/cpp/messagebus/remote_client_connection.cpp @@ -1,143 +1,143 @@ -#include "remote_client_connection.h" - +#include "remote_client_connection.h" + #include "mb_lwtrace.h" #include "network.h" -#include "remote_client_session.h" - +#include "remote_client_session.h" + #include <library/cpp/messagebus/actor/executor.h> #include <library/cpp/messagebus/actor/temp_tls_vector.h> - + #include <util/generic/cast.h> #include <util/thread/singleton.h> - -LWTRACE_USING(LWTRACE_MESSAGEBUS_PROVIDER) - -using namespace NActor; -using namespace NBus; -using namespace NBus::NPrivate; - -TRemoteClientConnection::TRemoteClientConnection(TRemoteClientSessionPtr session, ui64 id, TNetAddr addr) - : TRemoteConnection(session.Get(), id, addr) - , ClientHandler(GetSession()->ClientHandler) -{ + +LWTRACE_USING(LWTRACE_MESSAGEBUS_PROVIDER) + +using namespace NActor; +using namespace NBus; +using namespace NBus::NPrivate; + +TRemoteClientConnection::TRemoteClientConnection(TRemoteClientSessionPtr session, ui64 id, TNetAddr addr) + : TRemoteConnection(session.Get(), id, addr) + , ClientHandler(GetSession()->ClientHandler) +{ Y_VERIFY(addr.GetPort() > 0, "must connect to non-zero port"); - - ScheduleWrite(); -} - -TRemoteClientSession* TRemoteClientConnection::GetSession() { - return CheckedCast<TRemoteClientSession*>(Session.Get()); -} - -TBusMessage* TRemoteClientConnection::PopAck(TBusKey id) { - return AckMessages.Pop(id); -} - + + ScheduleWrite(); +} + +TRemoteClientSession* TRemoteClientConnection::GetSession() { + return CheckedCast<TRemoteClientSession*>(Session.Get()); +} + +TBusMessage* TRemoteClientConnection::PopAck(TBusKey id) { + return AckMessages.Pop(id); +} + SOCKET TRemoteClientConnection::CreateSocket(const TNetAddr& addr) { - SOCKET handle = socket(addr.Addr()->sa_family, SOCK_STREAM, 0); + SOCKET handle = socket(addr.Addr()->sa_family, SOCK_STREAM, 0); Y_VERIFY(handle != INVALID_SOCKET, "failed to create socket: %s", LastSystemErrorText()); - - TSocketHolder s(handle); - - SetNonBlock(s, true); - SetNoDelay(s, Config.TcpNoDelay); - SetSockOptTcpCork(s, Config.TcpCork); - SetCloseOnExec(s, true); - SetKeepAlive(s, true); - if (Config.SocketRecvBufferSize != 0) { - SetInputBuffer(s, Config.SocketRecvBufferSize); - } - if (Config.SocketSendBufferSize != 0) { - SetOutputBuffer(s, Config.SocketSendBufferSize); - } - if (Config.SocketToS >= 0) { - SetSocketToS(s, &addr, Config.SocketToS); - } - - return s.Release(); -} - -void TRemoteClientConnection::TryConnect() { - if (AtomicGet(WriterData.Down)) { - return; - } + + TSocketHolder s(handle); + + SetNonBlock(s, true); + SetNoDelay(s, Config.TcpNoDelay); + SetSockOptTcpCork(s, Config.TcpCork); + SetCloseOnExec(s, true); + SetKeepAlive(s, true); + if (Config.SocketRecvBufferSize != 0) { + SetInputBuffer(s, Config.SocketRecvBufferSize); + } + if (Config.SocketSendBufferSize != 0) { + SetOutputBuffer(s, Config.SocketSendBufferSize); + } + if (Config.SocketToS >= 0) { + SetSocketToS(s, &addr, Config.SocketToS); + } + + return s.Release(); +} + +void TRemoteClientConnection::TryConnect() { + if (AtomicGet(WriterData.Down)) { + return; + } Y_VERIFY(!WriterData.Status.Connected); - - TInstant now = TInstant::Now(); - - if (!WriterData.Channel) { - if ((now - LastConnectAttempt) < TDuration::MilliSeconds(Config.RetryInterval)) { + + TInstant now = TInstant::Now(); + + if (!WriterData.Channel) { + if ((now - LastConnectAttempt) < TDuration::MilliSeconds(Config.RetryInterval)) { DropEnqueuedData(MESSAGE_CONNECT_FAILED, MESSAGE_CONNECT_FAILED); - return; - } - LastConnectAttempt = now; - - TSocket connectSocket(CreateSocket(PeerAddr)); - WriterData.SetChannel(Session->WriteEventLoop.Register(connectSocket, this, WriteCookie)); - } - + return; + } + LastConnectAttempt = now; + + TSocket connectSocket(CreateSocket(PeerAddr)); + WriterData.SetChannel(Session->WriteEventLoop.Register(connectSocket, this, WriteCookie)); + } + if (BeforeSendQueue.IsEmpty() && WriterData.SendQueue.Empty() && !Config.ReconnectWhenIdle) { // TryConnect is called from Writer::Act, which is called in cycle // from session's ScheduleTimeoutMessages via Cron. This prevent these excessive connects. return; } - ++WriterData.Status.ConnectSyscalls; - - int ret = connect(WriterData.Channel->GetSocket(), PeerAddr.Addr(), PeerAddr.Len()); - int err = ret ? LastSystemError() : 0; - - if (!ret || (ret && err == EISCONN)) { - WriterData.Status.ConnectTime = now; - ++WriterData.SocketVersion; - - WriterData.Channel->DisableWrite(); - WriterData.Status.Connected = true; - AtomicSet(ReturnConnectFailedImmediately, false); - - WriterData.Status.MyAddr = TNetAddr(GetSockAddr(WriterData.Channel->GetSocket())); - - TSocket readSocket = WriterData.Channel->GetSocketPtr(); - - ReaderGetSocketQueue()->EnqueueAndSchedule(TWriterToReaderSocketMessage(readSocket, WriterData.SocketVersion)); - - FireClientConnectionEvent(TClientConnectionEvent::CONNECTED); - - ScheduleWrite(); - } else { - if (WouldBlock() || err == EALREADY) { - WriterData.Channel->EnableWrite(); - } else { - WriterData.DropChannel(); - WriterData.Status.MyAddr = TNetAddr(); - WriterData.Status.Connected = false; - WriterData.Status.ConnectError = err; - + ++WriterData.Status.ConnectSyscalls; + + int ret = connect(WriterData.Channel->GetSocket(), PeerAddr.Addr(), PeerAddr.Len()); + int err = ret ? LastSystemError() : 0; + + if (!ret || (ret && err == EISCONN)) { + WriterData.Status.ConnectTime = now; + ++WriterData.SocketVersion; + + WriterData.Channel->DisableWrite(); + WriterData.Status.Connected = true; + AtomicSet(ReturnConnectFailedImmediately, false); + + WriterData.Status.MyAddr = TNetAddr(GetSockAddr(WriterData.Channel->GetSocket())); + + TSocket readSocket = WriterData.Channel->GetSocketPtr(); + + ReaderGetSocketQueue()->EnqueueAndSchedule(TWriterToReaderSocketMessage(readSocket, WriterData.SocketVersion)); + + FireClientConnectionEvent(TClientConnectionEvent::CONNECTED); + + ScheduleWrite(); + } else { + if (WouldBlock() || err == EALREADY) { + WriterData.Channel->EnableWrite(); + } else { + WriterData.DropChannel(); + WriterData.Status.MyAddr = TNetAddr(); + WriterData.Status.Connected = false; + WriterData.Status.ConnectError = err; + DropEnqueuedData(MESSAGE_CONNECT_FAILED, MESSAGE_CONNECT_FAILED); - } - } -} - -void TRemoteClientConnection::HandleEvent(SOCKET socket, void* cookie) { + } + } +} + +void TRemoteClientConnection::HandleEvent(SOCKET socket, void* cookie) { Y_UNUSED(socket); Y_ASSERT(cookie == WriteCookie || cookie == ReadCookie); - if (cookie == ReadCookie) { - ScheduleRead(); - } else { - ScheduleWrite(); - } -} - -void TRemoteClientConnection::WriterFillStatus() { - TRemoteConnection::WriterFillStatus(); - WriterData.Status.AckMessagesSize = AckMessages.Size(); -} - -void TRemoteClientConnection::BeforeTryWrite() { - ProcessReplyQueue(); - TimeoutMessages(); -} - + if (cookie == ReadCookie) { + ScheduleRead(); + } else { + ScheduleWrite(); + } +} + +void TRemoteClientConnection::WriterFillStatus() { + TRemoteConnection::WriterFillStatus(); + WriterData.Status.AckMessagesSize = AckMessages.Size(); +} + +void TRemoteClientConnection::BeforeTryWrite() { + ProcessReplyQueue(); + TimeoutMessages(); +} + namespace NBus { namespace NPrivate { class TInvokeOnReply: public IWorkItem { @@ -145,7 +145,7 @@ namespace NBus { TRemoteClientSession* RemoteClientSession; TNonDestroyingHolder<TBusMessage> Request; TBusMessagePtrAndHeader Response; - + public: TInvokeOnReply(TRemoteClientSession* session, TNonDestroyingAutoPtr<TBusMessage> request, TBusMessagePtrAndHeader& response) @@ -154,7 +154,7 @@ namespace NBus { { Response.Swap(response); } - + void DoWork() override { THolder<TInvokeOnReply> holder(this); RemoteClientSession->ReleaseInFlightAndCallOnReply(Request.Release(), Response); @@ -162,182 +162,182 @@ namespace NBus { RemoteClientSession->JobCount.Decrement(); } }; - + } } - -void TRemoteClientConnection::ProcessReplyQueue() { - if (AtomicGet(WriterData.Down)) { - return; - } - - bool executeInWorkerPool = Session->Config.ExecuteOnReplyInWorkerPool; - - TTempTlsVector<TBusMessagePtrAndHeader, void, TVectorSwaps> replyQueueTemp; - TTempTlsVector< ::NActor::IWorkItem*> workQueueTemp; - - ReplyQueue.DequeueAllSingleConsumer(replyQueueTemp.GetVector()); - if (executeInWorkerPool) { - workQueueTemp.GetVector()->reserve(replyQueueTemp.GetVector()->size()); - } - + +void TRemoteClientConnection::ProcessReplyQueue() { + if (AtomicGet(WriterData.Down)) { + return; + } + + bool executeInWorkerPool = Session->Config.ExecuteOnReplyInWorkerPool; + + TTempTlsVector<TBusMessagePtrAndHeader, void, TVectorSwaps> replyQueueTemp; + TTempTlsVector< ::NActor::IWorkItem*> workQueueTemp; + + ReplyQueue.DequeueAllSingleConsumer(replyQueueTemp.GetVector()); + if (executeInWorkerPool) { + workQueueTemp.GetVector()->reserve(replyQueueTemp.GetVector()->size()); + } + for (auto& resp : *replyQueueTemp.GetVector()) { TBusMessage* req = PopAck(resp.Header.Id); - - if (!req) { + + if (!req) { WriterErrorMessage(resp.MessagePtr.Release(), MESSAGE_UNKNOWN); - continue; - } - - if (executeInWorkerPool) { + continue; + } + + if (executeInWorkerPool) { workQueueTemp.GetVector()->push_back(new TInvokeOnReply(GetSession(), req, resp)); - } else { + } else { GetSession()->ReleaseInFlightAndCallOnReply(req, resp); - } - } - - if (executeInWorkerPool) { - Session->JobCount.Add(workQueueTemp.GetVector()->size()); - Session->Queue->EnqueueWork(*workQueueTemp.GetVector()); - } -} - -void TRemoteClientConnection::TimeoutMessages() { - if (!TimeToTimeoutMessages.FetchTask()) { - return; - } - - TMessagesPtrs timedOutMessages; - - TInstant sendDeadline; - TInstant ackDeadline; - if (IsReturnConnectFailedImmediately()) { - sendDeadline = TInstant::Max(); - ackDeadline = TInstant::Max(); - } else { - TInstant now = TInstant::Now(); - sendDeadline = now - TDuration::MilliSeconds(Session->Config.SendTimeout); - ackDeadline = now - TDuration::MilliSeconds(Session->Config.TotalTimeout); - } - - { - TMessagesPtrs temp; - WriterData.SendQueue.Timeout(sendDeadline, &temp); - timedOutMessages.insert(timedOutMessages.end(), temp.begin(), temp.end()); - } - - // Ignores message that is being written currently (that is stored - // in WriteMessage). It is not a big problem, because after written - // to the network, message will be placed to the AckMessages queue, - // and timed out on the next iteration of this procedure. - - { - TMessagesPtrs temp; - AckMessages.Timeout(ackDeadline, &temp); - timedOutMessages.insert(timedOutMessages.end(), temp.begin(), temp.end()); - } - - ResetOneWayFlag(timedOutMessages); - - GetSession()->ReleaseInFlight(timedOutMessages); - WriterErrorMessages(timedOutMessages, MESSAGE_TIMEOUT); -} - -void TRemoteClientConnection::ScheduleTimeoutMessages() { - TimeToTimeoutMessages.AddTask(); - ScheduleWrite(); -} - + } + } + + if (executeInWorkerPool) { + Session->JobCount.Add(workQueueTemp.GetVector()->size()); + Session->Queue->EnqueueWork(*workQueueTemp.GetVector()); + } +} + +void TRemoteClientConnection::TimeoutMessages() { + if (!TimeToTimeoutMessages.FetchTask()) { + return; + } + + TMessagesPtrs timedOutMessages; + + TInstant sendDeadline; + TInstant ackDeadline; + if (IsReturnConnectFailedImmediately()) { + sendDeadline = TInstant::Max(); + ackDeadline = TInstant::Max(); + } else { + TInstant now = TInstant::Now(); + sendDeadline = now - TDuration::MilliSeconds(Session->Config.SendTimeout); + ackDeadline = now - TDuration::MilliSeconds(Session->Config.TotalTimeout); + } + + { + TMessagesPtrs temp; + WriterData.SendQueue.Timeout(sendDeadline, &temp); + timedOutMessages.insert(timedOutMessages.end(), temp.begin(), temp.end()); + } + + // Ignores message that is being written currently (that is stored + // in WriteMessage). It is not a big problem, because after written + // to the network, message will be placed to the AckMessages queue, + // and timed out on the next iteration of this procedure. + + { + TMessagesPtrs temp; + AckMessages.Timeout(ackDeadline, &temp); + timedOutMessages.insert(timedOutMessages.end(), temp.begin(), temp.end()); + } + + ResetOneWayFlag(timedOutMessages); + + GetSession()->ReleaseInFlight(timedOutMessages); + WriterErrorMessages(timedOutMessages, MESSAGE_TIMEOUT); +} + +void TRemoteClientConnection::ScheduleTimeoutMessages() { + TimeToTimeoutMessages.AddTask(); + ScheduleWrite(); +} + void TRemoteClientConnection::ReaderProcessMessageUnknownVersion(TArrayRef<const char>) { - LWPROBE(Error, ToString(MESSAGE_INVALID_VERSION), ToString(PeerAddr), ""); - ReaderData.Status.Incremental.StatusCounter[MESSAGE_INVALID_VERSION] += 1; - // TODO: close connection + LWPROBE(Error, ToString(MESSAGE_INVALID_VERSION), ToString(PeerAddr), ""); + ReaderData.Status.Incremental.StatusCounter[MESSAGE_INVALID_VERSION] += 1; + // TODO: close connection Y_FAIL("unknown message"); -} - -void TRemoteClientConnection::ClearOutgoingQueue(TMessagesPtrs& result, bool reconnect) { +} + +void TRemoteClientConnection::ClearOutgoingQueue(TMessagesPtrs& result, bool reconnect) { Y_ASSERT(result.empty()); - - TRemoteConnection::ClearOutgoingQueue(result, reconnect); - AckMessages.Clear(&result); - - ResetOneWayFlag(result); - GetSession()->ReleaseInFlight(result); -} - + + TRemoteConnection::ClearOutgoingQueue(result, reconnect); + AckMessages.Clear(&result); + + ResetOneWayFlag(result); + GetSession()->ReleaseInFlight(result); +} + void TRemoteClientConnection::MessageSent(TArrayRef<TBusMessagePtrAndHeader> messages) { for (auto& message : messages) { bool oneWay = message.LocalFlags & MESSAGE_ONE_WAY_INTERNAL; - - if (oneWay) { + + if (oneWay) { message.MessagePtr->LocalFlags &= ~MESSAGE_ONE_WAY_INTERNAL; - + TBusMessage* ackMsg = this->PopAck(message.Header.Id); - if (!ackMsg) { - // TODO: expired? - } - + if (!ackMsg) { + // TODO: expired? + } + if (ackMsg != message.MessagePtr.Get()) { - // TODO: non-unique id? - } - + // TODO: non-unique id? + } + GetSession()->ReleaseInFlight({message.MessagePtr.Get()}); ClientHandler->OnMessageSentOneWay(message.MessagePtr.Release()); - } else { + } else { ClientHandler->OnMessageSent(message.MessagePtr.Get()); AckMessages.Push(message); - } - } -} - -EMessageStatus TRemoteClientConnection::SendMessage(TBusMessage* req, bool wait) { - return SendMessageImpl(req, wait, false); -} - -EMessageStatus TRemoteClientConnection::SendMessageOneWay(TBusMessage* req, bool wait) { - return SendMessageImpl(req, wait, true); -} - -EMessageStatus TRemoteClientConnection::SendMessageImpl(TBusMessage* msg, bool wait, bool oneWay) { - msg->CheckClean(); - - if (Session->IsDown()) { - return MESSAGE_SHUTDOWN; - } - - if (wait) { + } + } +} + +EMessageStatus TRemoteClientConnection::SendMessage(TBusMessage* req, bool wait) { + return SendMessageImpl(req, wait, false); +} + +EMessageStatus TRemoteClientConnection::SendMessageOneWay(TBusMessage* req, bool wait) { + return SendMessageImpl(req, wait, true); +} + +EMessageStatus TRemoteClientConnection::SendMessageImpl(TBusMessage* msg, bool wait, bool oneWay) { + msg->CheckClean(); + + if (Session->IsDown()) { + return MESSAGE_SHUTDOWN; + } + + if (wait) { Y_VERIFY(!Session->Queue->GetExecutor()->IsInExecutorThread()); - GetSession()->ClientRemoteInFlight.Wait(); - } else { - if (!GetSession()->ClientRemoteInFlight.TryWait()) { - return MESSAGE_BUSY; - } - } - + GetSession()->ClientRemoteInFlight.Wait(); + } else { + if (!GetSession()->ClientRemoteInFlight.TryWait()) { + return MESSAGE_BUSY; + } + } + GetSession()->AcquireInFlight({msg}); - - EMessageStatus ret = MESSAGE_OK; - - if (oneWay) { - msg->LocalFlags |= MESSAGE_ONE_WAY_INTERNAL; - } - - msg->GetHeader()->SendTime = Now(); - - if (IsReturnConnectFailedImmediately()) { - ret = MESSAGE_CONNECT_FAILED; - goto clean; - } - - Send(msg); - - return MESSAGE_OK; -clean: - msg->LocalFlags &= ~MESSAGE_ONE_WAY_INTERNAL; + + EMessageStatus ret = MESSAGE_OK; + + if (oneWay) { + msg->LocalFlags |= MESSAGE_ONE_WAY_INTERNAL; + } + + msg->GetHeader()->SendTime = Now(); + + if (IsReturnConnectFailedImmediately()) { + ret = MESSAGE_CONNECT_FAILED; + goto clean; + } + + Send(msg); + + return MESSAGE_OK; +clean: + msg->LocalFlags &= ~MESSAGE_ONE_WAY_INTERNAL; GetSession()->ReleaseInFlight({msg}); - return ret; -} - -void TRemoteClientConnection::OpenConnection() { - // TODO -} + return ret; +} + +void TRemoteClientConnection::OpenConnection() { + // TODO +} diff --git a/library/cpp/messagebus/remote_client_connection.h b/library/cpp/messagebus/remote_client_connection.h index 124a37a07a..fe80b7d2f9 100644 --- a/library/cpp/messagebus/remote_client_connection.h +++ b/library/cpp/messagebus/remote_client_connection.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include "connection.h" -#include "local_tasks.h" +#include "local_tasks.h" #include "remote_client_session.h" -#include "remote_connection.h" - +#include "remote_connection.h" + #include <util/generic/object_counter.h> namespace NBus { @@ -13,53 +13,53 @@ namespace NBus { friend class TRemoteConnection; friend struct TBusSessionImpl; friend class TRemoteClientSession; - + private: TObjectCounter<TRemoteClientConnection> ObjectCounter; - + TSyncAckMessages AckMessages; - + TLocalTasks TimeToTimeoutMessages; - + IBusClientHandler* const ClientHandler; - + public: TRemoteClientConnection(TRemoteClientSessionPtr session, ui64 id, TNetAddr addr); - + inline TRemoteClientSession* GetSession(); - + SOCKET CreateSocket(const TNetAddr& addr); - + void TryConnect() override; - + void HandleEvent(SOCKET socket, void* cookie) override; - + TBusMessage* PopAck(TBusKey id); - + void WriterFillStatus() override; - + void ClearOutgoingQueue(TMessagesPtrs& result, bool reconnect) override; - + void BeforeTryWrite() override; - + void ProcessReplyQueue(); - + void MessageSent(TArrayRef<TBusMessagePtrAndHeader> messages) override; - + void TimeoutMessages(); - + void ScheduleTimeoutMessages(); - + void ReaderProcessMessageUnknownVersion(TArrayRef<const char> dataRef) override; - + EMessageStatus SendMessage(TBusMessage* pMes, bool wait) override; - + EMessageStatus SendMessageOneWay(TBusMessage* pMes, bool wait) override; - + EMessageStatus SendMessageImpl(TBusMessage*, bool wait, bool oneWay); - + void OpenConnection() override; }; - + } } diff --git a/library/cpp/messagebus/remote_client_session.cpp b/library/cpp/messagebus/remote_client_session.cpp index 70c20b9063..3bc421944f 100644 --- a/library/cpp/messagebus/remote_client_session.cpp +++ b/library/cpp/messagebus/remote_client_session.cpp @@ -1,127 +1,127 @@ -#include "remote_client_session.h" +#include "remote_client_session.h" #include "mb_lwtrace.h" -#include "remote_client_connection.h" - +#include "remote_client_connection.h" + #include <library/cpp/messagebus/scheduler/scheduler.h> #include <util/generic/cast.h> #include <util/system/defaults.h> - -LWTRACE_USING(LWTRACE_MESSAGEBUS_PROVIDER) - -using namespace NBus; -using namespace NBus::NPrivate; - -TRemoteClientSession::TRemoteClientSession(TBusMessageQueue* queue, + +LWTRACE_USING(LWTRACE_MESSAGEBUS_PROVIDER) + +using namespace NBus; +using namespace NBus::NPrivate; + +TRemoteClientSession::TRemoteClientSession(TBusMessageQueue* queue, TBusProtocol* proto, IBusClientHandler* handler, const TBusClientSessionConfig& config, const TString& name) - : TBusSessionImpl(true, queue, proto, handler, config, name) - , ClientRemoteInFlight(config.MaxInFlight, "ClientRemoteInFlight") - , ClientHandler(handler) + : TBusSessionImpl(true, queue, proto, handler, config, name) + , ClientRemoteInFlight(config.MaxInFlight, "ClientRemoteInFlight") + , ClientHandler(handler) { } -TRemoteClientSession::~TRemoteClientSession() { - //Cerr << "~TRemoteClientSession" << Endl; -} - -void TRemoteClientSession::OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) { +TRemoteClientSession::~TRemoteClientSession() { + //Cerr << "~TRemoteClientSession" << Endl; +} + +void TRemoteClientSession::OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) { TAutoPtr<TVectorSwaps<TBusMessagePtrAndHeader>> temp(new TVectorSwaps<TBusMessagePtrAndHeader>); - temp->swap(newMsg); - c->ReplyQueue.EnqueueAll(temp); - c->ScheduleWrite(); + temp->swap(newMsg); + c->ReplyQueue.EnqueueAll(temp); + c->ScheduleWrite(); } - -EMessageStatus TRemoteClientSession::SendMessageImpl(TBusMessage* msg, const TNetAddr* addr, bool wait, bool oneWay) { + +EMessageStatus TRemoteClientSession::SendMessageImpl(TBusMessage* msg, const TNetAddr* addr, bool wait, bool oneWay) { if (Y_UNLIKELY(IsDown())) { - return MESSAGE_SHUTDOWN; - } - - TBusSocketAddr resolvedAddr; - EMessageStatus ret = GetMessageDestination(msg, addr, &resolvedAddr); - if (ret != MESSAGE_OK) { - return ret; - } - - msg->ReplyTo = resolvedAddr; - + return MESSAGE_SHUTDOWN; + } + + TBusSocketAddr resolvedAddr; + EMessageStatus ret = GetMessageDestination(msg, addr, &resolvedAddr); + if (ret != MESSAGE_OK) { + return ret; + } + + msg->ReplyTo = resolvedAddr; + TRemoteConnectionPtr c = ((TBusSessionImpl*)this)->GetConnection(resolvedAddr, true); Y_ASSERT(!!c); - - return CheckedCast<TRemoteClientConnection*>(c.Get())->SendMessageImpl(msg, wait, oneWay); -} - -EMessageStatus TRemoteClientSession::SendMessage(TBusMessage* msg, const TNetAddr* addr, bool wait) { - return SendMessageImpl(msg, addr, wait, false); -} - -EMessageStatus TRemoteClientSession::SendMessageOneWay(TBusMessage* pMes, const TNetAddr* addr, bool wait) { - return SendMessageImpl(pMes, addr, wait, true); -} - + + return CheckedCast<TRemoteClientConnection*>(c.Get())->SendMessageImpl(msg, wait, oneWay); +} + +EMessageStatus TRemoteClientSession::SendMessage(TBusMessage* msg, const TNetAddr* addr, bool wait) { + return SendMessageImpl(msg, addr, wait, false); +} + +EMessageStatus TRemoteClientSession::SendMessageOneWay(TBusMessage* pMes, const TNetAddr* addr, bool wait) { + return SendMessageImpl(pMes, addr, wait, true); +} + int TRemoteClientSession::GetInFlight() const noexcept { - return ClientRemoteInFlight.GetCurrent(); -} - -void TRemoteClientSession::FillStatus() { - TBusSessionImpl::FillStatus(); - - StatusData.Status.InFlightCount = ClientRemoteInFlight.GetCurrent(); - StatusData.Status.InputPaused = false; -} - + return ClientRemoteInFlight.GetCurrent(); +} + +void TRemoteClientSession::FillStatus() { + TBusSessionImpl::FillStatus(); + + StatusData.Status.InFlightCount = ClientRemoteInFlight.GetCurrent(); + StatusData.Status.InputPaused = false; +} + void TRemoteClientSession::AcquireInFlight(TArrayRef<TBusMessage* const> messages) { for (auto message : messages) { Y_ASSERT(!(message->LocalFlags & MESSAGE_IN_FLIGHT_ON_CLIENT)); message->LocalFlags |= MESSAGE_IN_FLIGHT_ON_CLIENT; - } - ClientRemoteInFlight.IncrementMultiple(messages.size()); -} - + } + ClientRemoteInFlight.IncrementMultiple(messages.size()); +} + void TRemoteClientSession::ReleaseInFlight(TArrayRef<TBusMessage* const> messages) { for (auto message : messages) { Y_ASSERT(message->LocalFlags & MESSAGE_IN_FLIGHT_ON_CLIENT); message->LocalFlags &= ~MESSAGE_IN_FLIGHT_ON_CLIENT; - } - ClientRemoteInFlight.ReleaseMultiple(messages.size()); -} - -void TRemoteClientSession::ReleaseInFlightAndCallOnReply(TNonDestroyingAutoPtr<TBusMessage> request, TBusMessagePtrAndHeader& response) { + } + ClientRemoteInFlight.ReleaseMultiple(messages.size()); +} + +void TRemoteClientSession::ReleaseInFlightAndCallOnReply(TNonDestroyingAutoPtr<TBusMessage> request, TBusMessagePtrAndHeader& response) { ReleaseInFlight({request.Get()}); if (Y_UNLIKELY(AtomicGet(Down))) { - InvokeOnError(request, MESSAGE_SHUTDOWN); - InvokeOnError(response.MessagePtr.Release(), MESSAGE_SHUTDOWN); - - TRemoteConnectionReaderIncrementalStatus counter; - LWPROBE(Error, ToString(MESSAGE_SHUTDOWN), "", ""); - counter.StatusCounter[MESSAGE_SHUTDOWN] += 1; - GetDeadConnectionReaderStatusQueue()->EnqueueAndSchedule(counter); - } else { - TWhatThreadDoesPushPop pp("OnReply"); - ClientHandler->OnReply(request, response.MessagePtr.Release()); - } -} - + InvokeOnError(request, MESSAGE_SHUTDOWN); + InvokeOnError(response.MessagePtr.Release(), MESSAGE_SHUTDOWN); + + TRemoteConnectionReaderIncrementalStatus counter; + LWPROBE(Error, ToString(MESSAGE_SHUTDOWN), "", ""); + counter.StatusCounter[MESSAGE_SHUTDOWN] += 1; + GetDeadConnectionReaderStatusQueue()->EnqueueAndSchedule(counter); + } else { + TWhatThreadDoesPushPop pp("OnReply"); + ClientHandler->OnReply(request, response.MessagePtr.Release()); + } +} + EMessageStatus TRemoteClientSession::GetMessageDestination(TBusMessage* mess, const TNetAddr* addrp, TBusSocketAddr* dest) { - if (addrp) { - *dest = *addrp; - } else { - TNetAddr tmp; - EMessageStatus ret = const_cast<TBusProtocol*>(GetProto())->GetDestination(this, mess, GetQueue()->GetLocator(), &tmp); - if (ret != MESSAGE_OK) { - return ret; - } - *dest = tmp; - } - return MESSAGE_OK; -} - -void TRemoteClientSession::OpenConnection(const TNetAddr& addr) { - GetConnection(addr)->OpenConnection(); -} - -TBusClientConnectionPtr TRemoteClientSession::GetConnection(const TNetAddr& addr) { - // TODO: GetConnection should not open + if (addrp) { + *dest = *addrp; + } else { + TNetAddr tmp; + EMessageStatus ret = const_cast<TBusProtocol*>(GetProto())->GetDestination(this, mess, GetQueue()->GetLocator(), &tmp); + if (ret != MESSAGE_OK) { + return ret; + } + *dest = tmp; + } + return MESSAGE_OK; +} + +void TRemoteClientSession::OpenConnection(const TNetAddr& addr) { + GetConnection(addr)->OpenConnection(); +} + +TBusClientConnectionPtr TRemoteClientSession::GetConnection(const TNetAddr& addr) { + // TODO: GetConnection should not open return CheckedCast<TRemoteClientConnection*>(((TBusSessionImpl*)this)->GetConnection(addr, true).Get()); -} +} diff --git a/library/cpp/messagebus/remote_client_session.h b/library/cpp/messagebus/remote_client_session.h index 8268a6c0a3..7160d0dae9 100644 --- a/library/cpp/messagebus/remote_client_session.h +++ b/library/cpp/messagebus/remote_client_session.h @@ -1,8 +1,8 @@ #pragma once -#include "remote_client_session_semaphore.h" -#include "session_impl.h" - +#include "remote_client_session_semaphore.h" +#include "session_impl.h" + #include <util/generic/array_ref.h> #include <util/generic/object_counter.h> @@ -10,7 +10,7 @@ #pragma warning(push) #pragma warning(disable : 4250) // 'NBus::NPrivate::TRemoteClientSession' : inherits 'NBus::NPrivate::TBusSessionImpl::NBus::NPrivate::TBusSessionImpl::GetConfig' via dominance #endif - + namespace NBus { namespace NPrivate { using TRemoteClientSessionPtr = TIntrusivePtr<TRemoteClientSession>; @@ -18,39 +18,39 @@ namespace NBus { class TRemoteClientSession: public TBusClientSession, public TBusSessionImpl { friend class TRemoteClientConnection; friend class TInvokeOnReply; - + public: TObjectCounter<TRemoteClientSession> ObjectCounter; - + TRemoteClientSessionSemaphore ClientRemoteInFlight; IBusClientHandler* const ClientHandler; - + public: TRemoteClientSession(TBusMessageQueue* queue, TBusProtocol* proto, IBusClientHandler* handler, const TBusSessionConfig& config, const TString& name); - + ~TRemoteClientSession() override; - + void OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) override; - + EMessageStatus SendMessageImpl(TBusMessage* msg, const TNetAddr* addr, bool wait, bool oneWay); EMessageStatus SendMessage(TBusMessage* msg, const TNetAddr* addr = nullptr, bool wait = false) override; EMessageStatus SendMessageOneWay(TBusMessage* msg, const TNetAddr* addr = nullptr, bool wait = false) override; - + int GetInFlight() const noexcept override; void FillStatus() override; void AcquireInFlight(TArrayRef<TBusMessage* const> messages); void ReleaseInFlight(TArrayRef<TBusMessage* const> messages); void ReleaseInFlightAndCallOnReply(TNonDestroyingAutoPtr<TBusMessage> request, TBusMessagePtrAndHeader& response); - + EMessageStatus GetMessageDestination(TBusMessage* mess, const TNetAddr* addrp, TBusSocketAddr* dest); - + void OpenConnection(const TNetAddr&) override; - + TBusClientConnectionPtr GetConnection(const TNetAddr&) override; }; - + #ifdef _MSC_VER #pragma warning(pop) #endif diff --git a/library/cpp/messagebus/remote_client_session_semaphore.cpp b/library/cpp/messagebus/remote_client_session_semaphore.cpp index f1d02cc5c2..f877ed4257 100644 --- a/library/cpp/messagebus/remote_client_session_semaphore.cpp +++ b/library/cpp/messagebus/remote_client_session_semaphore.cpp @@ -1,67 +1,67 @@ #include "remote_client_session_semaphore.h" #include <util/stream/output.h> -#include <util/system/yassert.h> - -using namespace NBus; -using namespace NBus::NPrivate; - -TRemoteClientSessionSemaphore::TRemoteClientSessionSemaphore(TAtomicBase limit, const char* name) - : Name(name) - , Limit(limit) - , Current(0) - , StopSignal(0) -{ +#include <util/system/yassert.h> + +using namespace NBus; +using namespace NBus::NPrivate; + +TRemoteClientSessionSemaphore::TRemoteClientSessionSemaphore(TAtomicBase limit, const char* name) + : Name(name) + , Limit(limit) + , Current(0) + , StopSignal(0) +{ Y_VERIFY(limit > 0, "limit must be > 0"); Y_UNUSED(Name); -} - +} + TRemoteClientSessionSemaphore::~TRemoteClientSessionSemaphore() { Y_VERIFY(AtomicGet(Current) == 0); -} - -bool TRemoteClientSessionSemaphore::TryAcquire() { - if (!TryWait()) { - return false; - } - - AtomicIncrement(Current); - return true; -} - -bool TRemoteClientSessionSemaphore::TryWait() { - if (AtomicGet(Current) < Limit) - return true; +} + +bool TRemoteClientSessionSemaphore::TryAcquire() { + if (!TryWait()) { + return false; + } + + AtomicIncrement(Current); + return true; +} + +bool TRemoteClientSessionSemaphore::TryWait() { + if (AtomicGet(Current) < Limit) + return true; if (Y_UNLIKELY(AtomicGet(StopSignal))) - return true; - return false; -} - -void TRemoteClientSessionSemaphore::Acquire() { - Wait(); - - Increment(); -} - -void TRemoteClientSessionSemaphore::Increment() { - IncrementMultiple(1); -} - -void TRemoteClientSessionSemaphore::IncrementMultiple(TAtomicBase count) { - AtomicAdd(Current, count); - Updated(); -} - -void TRemoteClientSessionSemaphore::Release() { - ReleaseMultiple(1); -} - -void TRemoteClientSessionSemaphore::ReleaseMultiple(TAtomicBase count) { - AtomicSub(Current, count); - Updated(); -} - -void TRemoteClientSessionSemaphore::Stop() { - AtomicSet(StopSignal, 1); - Updated(); -} + return true; + return false; +} + +void TRemoteClientSessionSemaphore::Acquire() { + Wait(); + + Increment(); +} + +void TRemoteClientSessionSemaphore::Increment() { + IncrementMultiple(1); +} + +void TRemoteClientSessionSemaphore::IncrementMultiple(TAtomicBase count) { + AtomicAdd(Current, count); + Updated(); +} + +void TRemoteClientSessionSemaphore::Release() { + ReleaseMultiple(1); +} + +void TRemoteClientSessionSemaphore::ReleaseMultiple(TAtomicBase count) { + AtomicSub(Current, count); + Updated(); +} + +void TRemoteClientSessionSemaphore::Stop() { + AtomicSet(StopSignal, 1); + Updated(); +} diff --git a/library/cpp/messagebus/remote_client_session_semaphore.h b/library/cpp/messagebus/remote_client_session_semaphore.h index 3bb86213c2..286ca3c86f 100644 --- a/library/cpp/messagebus/remote_client_session_semaphore.h +++ b/library/cpp/messagebus/remote_client_session_semaphore.h @@ -1,30 +1,30 @@ -#pragma once - +#pragma once + #include "cc_semaphore.h" -#include <util/generic/noncopyable.h> +#include <util/generic/noncopyable.h> #include <util/system/atomic.h> #include <util/system/condvar.h> -#include <util/system/mutex.h> - +#include <util/system/mutex.h> + namespace NBus { namespace NPrivate { class TRemoteClientSessionSemaphore: public TComplexConditionSemaphore<TRemoteClientSessionSemaphore> { private: const char* const Name; - + TAtomicBase const Limit; TAtomic Current; TAtomic StopSignal; - + public: TRemoteClientSessionSemaphore(TAtomicBase limit, const char* name = "unnamed"); ~TRemoteClientSessionSemaphore(); - + TAtomicBase GetCurrent() const { return AtomicGet(Current); } - + void Acquire(); bool TryAcquire(); void Increment(); @@ -33,10 +33,10 @@ namespace NBus { void Release(); void ReleaseMultiple(TAtomicBase count); void Stop(); - + private: void CheckNeedToUnlock(); }; - + } } diff --git a/library/cpp/messagebus/remote_connection.cpp b/library/cpp/messagebus/remote_connection.cpp index 2e14d78eb4..22932569db 100644 --- a/library/cpp/messagebus/remote_connection.cpp +++ b/library/cpp/messagebus/remote_connection.cpp @@ -1,11 +1,11 @@ #include "remote_connection.h" - -#include "key_value_printer.h" + +#include "key_value_printer.h" #include "mb_lwtrace.h" #include "network.h" -#include "remote_client_connection.h" -#include "remote_client_session.h" -#include "remote_server_session.h" +#include "remote_client_connection.h" +#include "remote_client_session.h" +#include "remote_server_session.h" #include "session_impl.h" #include <library/cpp/messagebus/actor/what_thread_does.h> @@ -14,12 +14,12 @@ #include <util/network/init.h> #include <util/system/atomic.h> -LWTRACE_USING(LWTRACE_MESSAGEBUS_PROVIDER) - -using namespace NActor; -using namespace NBus; -using namespace NBus::NPrivate; - +LWTRACE_USING(LWTRACE_MESSAGEBUS_PROVIDER) + +using namespace NActor; +using namespace NBus; +using namespace NBus::NPrivate; + namespace NBus { namespace NPrivate { TRemoteConnection::TRemoteConnection(TRemoteSessionPtr session, ui64 connectionId, TNetAddr addr) @@ -46,7 +46,7 @@ namespace NBus { ReaderData.Status.ConnectionId = connectionId; const TInstant now = TInstant::Now(); - + WriterFillStatus(); GranStatus.Writer.Update(WriterData.Status, now, true); @@ -56,7 +56,7 @@ namespace NBus { TRemoteConnection::~TRemoteConnection() { Y_VERIFY(ReplyQueue.IsEmpty()); } - + TRemoteConnection::TWriterData::TWriterData() : Down(0) , SocketVersion(0) @@ -65,7 +65,7 @@ namespace NBus { , State(WRITER_FILLING) { } - + TRemoteConnection::TWriterData::~TWriterData() { Y_VERIFY(AtomicGet(Down)); Y_VERIFY(SendQueue.Empty()); @@ -76,7 +76,7 @@ namespace NBus { return (MoreBytes = left >= bytes ? 0 : bytes - left) == 0; } - + void TRemoteConnection::TWriterData::SetChannel(NEventLoop::TChannelPtr channel) { Y_VERIFY(!Channel, "must not have channel"); Y_VERIFY(Buffer.GetBuffer().Empty() && Buffer.LeftSize() == 0, "buffer must be empty"); @@ -95,11 +95,11 @@ namespace NBus { Channel->Unregister(); Channel.Drop(); } - + Buffer.Reset(); State = WRITER_FILLING; } - + void TRemoteConnection::TReaderData::DropChannel() { // TODO: make Drop call Unregister if (!!Channel) { @@ -109,7 +109,7 @@ namespace NBus { Buffer.Reset(); Offset = 0; } - + TRemoteConnection::TReaderData::TReaderData() : Down(0) , SocketVersion(0) @@ -117,31 +117,31 @@ namespace NBus { , MoreBytes(0) { } - + TRemoteConnection::TReaderData::~TReaderData() { Y_VERIFY(AtomicGet(Down)); } - + void TRemoteConnection::Send(TNonDestroyingAutoPtr<TBusMessage> msg) { BeforeSendQueue.Enqueue(msg.Release()); AtomicIncrement(WriterData.InFlight); ScheduleWrite(); } - + void TRemoteConnection::ClearOutgoingQueue(TMessagesPtrs& result, bool reconnect) { if (!reconnect) { // Do not clear send queue if reconnecting WriterData.SendQueue.Clear(&result); } } - + void TRemoteConnection::Shutdown(EMessageStatus status) { ScheduleShutdown(status); ReaderData.ShutdownComplete.WaitI(); WriterData.ShutdownComplete.WaitI(); } - + void TRemoteConnection::TryConnect() { Y_FAIL("TryConnect is client connection only operation"); } @@ -158,27 +158,27 @@ namespace NBus { if (!WriterData.TimeToRotateCounters.FetchTask()) { return; } - + WriterData.Status.DurationCounterPrev = WriterData.Status.DurationCounter; Reset(WriterData.Status.DurationCounter); } - + void TRemoteConnection::WriterSendStatus(TInstant now, bool force) { GranStatus.Writer.Update(std::bind(&TRemoteConnection::WriterGetStatus, this), now, force); } - + void TRemoteConnection::ReaderSendStatus(TInstant now, bool force) { GranStatus.Reader.Update(std::bind(&TRemoteConnection::ReaderFillStatus, this), now, force); } - + const TRemoteConnectionReaderStatus& TRemoteConnection::ReaderFillStatus() { ReaderData.Status.BufferSize = ReaderData.Buffer.Capacity(); ReaderData.Status.QuotaMsg = QuotaMsg.Tokens(); ReaderData.Status.QuotaBytes = QuotaBytes.Tokens(); - + return ReaderData.Status; } - + void TRemoteConnection::ProcessItem(TReaderTag, ::NActor::TDefaultTag, TWriterToReaderSocketMessage readSocket) { if (AtomicGet(ReaderData.Down)) { ReaderData.Status.Fd = INVALID_SOCKET; @@ -189,48 +189,48 @@ namespace NBus { ReaderData.Status.Fd = readSocket.Socket; ReaderData.SocketVersion = readSocket.SocketVersion; - + if (readSocket.Socket != INVALID_SOCKET) { ReaderData.SetChannel(Session->ReadEventLoop.Register(readSocket.Socket, this, ReadCookie)); ReaderData.Channel->EnableRead(); } } - + void TRemoteConnection::ProcessItem(TWriterTag, TReconnectTag, ui32 socketVersion) { Y_VERIFY(socketVersion <= WriterData.SocketVersion, "something weird"); - + if (WriterData.SocketVersion != socketVersion) { return; } Y_VERIFY(WriterData.Status.Connected, "must be connected at this point"); Y_VERIFY(!!WriterData.Channel, "must have channel at this point"); - + WriterData.Status.Connected = false; WriterData.DropChannel(); WriterData.Status.MyAddr = TNetAddr(); ++WriterData.SocketVersion; LastConnectAttempt = TInstant(); - + TMessagesPtrs cleared; ClearOutgoingQueue(cleared, true); WriterErrorMessages(cleared, MESSAGE_DELIVERY_FAILED); - + FireClientConnectionEvent(TClientConnectionEvent::DISCONNECTED); - + ReaderGetSocketQueue()->EnqueueAndSchedule(TWriterToReaderSocketMessage(INVALID_SOCKET, WriterData.SocketVersion)); } - + void TRemoteConnection::ProcessItem(TWriterTag, TWakeReaderTag, ui32 awakeFlags) { WriterData.AwakeFlags |= awakeFlags; - + ReadQuotaWakeup(); } - + void TRemoteConnection::Act(TReaderTag) { TInstant now = TInstant::Now(); - + ReaderData.Status.Acts += 1; - + ReaderGetSocketQueue()->DequeueAllLikelyEmpty(); if (AtomicGet(ReaderData.Down)) { @@ -238,41 +238,41 @@ namespace NBus { ReaderProcessStatusDown(); ReaderData.ShutdownComplete.Signal(); - + } else if (!!ReaderData.Channel) { Y_ASSERT(ReaderData.ReadMessages.empty()); - + for (int i = 0;; ++i) { if (i == 100) { // perform other tasks GetReaderActor()->AddTaskFromActorLoop(); break; } - + if (NeedInterruptRead()) { ReaderData.Channel->EnableRead(); break; } - + if (!ReaderFillBuffer()) break; - + if (!ReaderProcessBuffer()) break; } - + ReaderFlushMessages(); } - + ReaderSendStatus(now); - } - + } + bool TRemoteConnection::QuotaAcquire(size_t msg, size_t bytes) { ui32 wakeFlags = 0; - + if (!QuotaMsg.Acquire(msg)) wakeFlags |= WAKE_QUOTA_MSG; - + else if (!QuotaBytes.Acquire(bytes)) wakeFlags |= WAKE_QUOTA_BYTES; @@ -380,7 +380,7 @@ namespace NBus { ReaderData.Buffer.ChopHead(ReaderData.Offset); ReaderData.Offset = 0; - + if (ReaderData.Buffer.Capacity() > MaxBufferSize && ReaderData.Buffer.Size() <= MaxBufferSize) { ReaderData.Status.Incremental.BufferDrops += 1; @@ -388,7 +388,7 @@ namespace NBus { // probably should use another constant temp.Reserve(Config.DefaultBufferSize); temp.Append(ReaderData.Buffer.Data(), ReaderData.Buffer.Size()); - + ReaderData.Buffer.Swap(temp); } @@ -398,14 +398,14 @@ namespace NBus { bool TRemoteConnection::ReaderFillBuffer() { if (!ReaderData.BufferMore()) return true; - + if (ReaderData.Buffer.Avail() == 0) { if (ReaderData.Buffer.Size() == 0) { ReaderData.Buffer.Reserve(Config.DefaultBufferSize); } else { ReaderData.Buffer.Reserve(ReaderData.Buffer.Size() * 2); } - } + } Y_ASSERT(ReaderData.Buffer.Avail() > 0); @@ -414,7 +414,7 @@ namespace NBus { TWhatThreadDoesPushPop pp("recv syscall"); bytes = SocketRecv(ReaderData.Channel->GetSocket(), TArrayRef<char>(ReaderData.Buffer.Pos(), ReaderData.Buffer.Avail())); } - + if (bytes < 0) { if (WouldBlock()) { ReaderData.Channel->EnableRead(); @@ -425,30 +425,30 @@ namespace NBus { return false; } } - + if (bytes == 0) { ReaderData.Channel->DisableRead(); // TODO: incorrect: it is possible that only input is shutdown, and output is available ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_DELIVERY_FAILED, false); return false; } - + ReaderData.Status.Incremental.NetworkOps += 1; - + ReaderData.Buffer.Advance(bytes); ReaderData.MoreBytes = 0; return true; } - + void TRemoteConnection::ClearBeforeSendQueue(EMessageStatus reason) { BeforeSendQueue.DequeueAll(std::bind(&TRemoteConnection::WriterBeforeWriteErrorMessage, this, std::placeholders::_1, reason)); } - + void TRemoteConnection::ClearReplyQueue(EMessageStatus reason) { TVectorSwaps<TBusMessagePtrAndHeader> replyQueueTemp; Y_ASSERT(replyQueueTemp.empty()); ReplyQueue.DequeueAllSingleConsumer(&replyQueueTemp); - + TVector<TBusMessage*> messages; for (TVectorSwaps<TBusMessagePtrAndHeader>::reverse_iterator message = replyQueueTemp.rbegin(); message != replyQueueTemp.rend(); ++message) { @@ -458,8 +458,8 @@ namespace NBus { WriterErrorMessages(messages, reason); replyQueueTemp.clear(); - } - + } + void TRemoteConnection::ProcessBeforeSendQueueMessage(TBusMessage* message, TInstant now) { // legacy clients expect this field to be set if (!Session->IsSource_) { @@ -482,7 +482,7 @@ namespace NBus { const TRemoteConnectionWriterStatus& TRemoteConnection::WriterGetStatus() { WriterRotateCounters(); WriterFillStatus(); - + return WriterData.Status; } @@ -496,40 +496,40 @@ namespace NBus { WriterData.Status.SendQueueSize = WriterData.SendQueue.Size(); WriterData.Status.State = WriterData.State; } - + void TRemoteConnection::WriterProcessStatusDown() { Session->GetDeadConnectionWriterStatusQueue()->EnqueueAndSchedule(WriterData.Status.Incremental); Reset(WriterData.Status.Incremental); } - + void TRemoteConnection::ReaderProcessStatusDown() { Session->GetDeadConnectionReaderStatusQueue()->EnqueueAndSchedule(ReaderData.Status.Incremental); Reset(ReaderData.Status.Incremental); } - + void TRemoteConnection::ProcessWriterDown() { if (!RemovedFromSession) { Session->GetRemoveConnectionQueue()->EnqueueAndSchedule(this); - + if (Session->IsSource_) { if (WriterData.Status.Connected) { FireClientConnectionEvent(TClientConnectionEvent::DISCONNECTED); } } - + LWPROBE(Disconnected, ToString(PeerAddr)); RemovedFromSession = true; } - + WriterData.DropChannel(); - + DropEnqueuedData(ShutdownReason, MESSAGE_SHUTDOWN); - + WriterProcessStatusDown(); - + WriterData.ShutdownComplete.Signal(); } - + void TRemoteConnection::DropEnqueuedData(EMessageStatus reason, EMessageStatus reasonForQueues) { ClearReplyQueue(reasonForQueues); ClearBeforeSendQueue(reasonForQueues); @@ -554,23 +554,23 @@ namespace NBus { void TRemoteConnection::BeforeTryWrite() { } - + void TRemoteConnection::Act(TWriterTag) { TInstant now = TInstant::Now(); - + WriterData.Status.Acts += 1; - + if (Y_UNLIKELY(AtomicGet(WriterData.Down))) { // dump status must work even if WriterDown WriterSendStatus(now, true); ProcessWriterDown(); return; - } - + } + ProcessBeforeSendQueue(now); - + BeforeTryWrite(); - + WriterFillInFlight(); WriterGetReconnectQueue()->DequeueAllLikelyEmpty(); @@ -587,43 +587,43 @@ namespace NBus { if (WriterData.State == WRITER_FILLING) { WriterFillBuffer(); - + if (WriterData.State == WRITER_FILLING) { WriterData.Channel->DisableWrite(); break; } - + Y_ASSERT(!WriterData.Buffer.Empty()); } - + if (WriterData.State == WRITER_FLUSHING) { WriterFlushBuffer(); - + if (WriterData.State == WRITER_FLUSHING) { break; } } } } - + WriterGetWakeQueue()->DequeueAllLikelyEmpty(); - + WriterSendStatus(now); } - + void TRemoteConnection::WriterFlushBuffer() { Y_ASSERT(WriterData.State == WRITER_FLUSHING); Y_ASSERT(!WriterData.Buffer.Empty()); - + WriterData.CorkUntil = TInstant::Zero(); - + while (!WriterData.Buffer.Empty()) { ssize_t bytes; { TWhatThreadDoesPushPop pp("send syscall"); bytes = SocketSend(WriterData.Channel->GetSocket(), TArrayRef<const char>(WriterData.Buffer.LeftPos(), WriterData.Buffer.Size())); } - + if (bytes < 0) { if (WouldBlock()) { WriterData.Channel->EnableWrite(); @@ -634,18 +634,18 @@ namespace NBus { return; } } - + WriterData.Status.Incremental.NetworkOps += 1; - + WriterData.Buffer.LeftProceed(bytes); } - + WriterData.Buffer.Clear(); if (WriterData.Buffer.Capacity() > MaxBufferSize) { WriterData.Status.Incremental.BufferDrops += 1; WriterData.Buffer.Reset(); } - + WriterData.State = WRITER_FILLING; } @@ -655,8 +655,8 @@ namespace NBus { } else { ScheduleShutdown(status); } - } - + } + void TRemoteConnection::ScheduleShutdown(EMessageStatus status) { ShutdownReason = status; @@ -673,20 +673,20 @@ namespace NBus { Y_VERIFY(buffer.Size() >= posForAssertion, "incorrect Serialize implementation, pos before serialize: %d, pos after serialize: %d", int(posForAssertion), int(buffer.Size())); - } - + } + namespace { inline void WriteHeader(const TBusHeader& header, TBuffer& data) { data.Reserve(data.Size() + sizeof(TBusHeader)); /// \todo hton instead of memcpy memcpy(data.Data() + data.Size(), &header, sizeof(TBusHeader)); data.Advance(sizeof(TBusHeader)); - } - + } + inline void WriteDummyHeader(TBuffer& data) { data.Resize(data.Size() + sizeof(TBusHeader)); } - + } void TRemoteConnection::SerializeMessage(TBusMessage* msg, TBuffer* data, TMessageCounter* counter) const { @@ -695,17 +695,17 @@ namespace NBus { size_t dataSize; bool compressionRequested = msg->IsCompressed(); - + if (compressionRequested) { TBuffer compdata; TBuffer plaindata; CallSerialize(msg, plaindata); - + dataSize = sizeof(TBusHeader) + plaindata.Size(); - + NCodecs::TCodecPtr c = Proto->GetTransportCodec(); c->Encode(TStringBuf{plaindata.data(), plaindata.size()}, compdata); - + if (compdata.Size() < plaindata.Size()) { plaindata.Clear(); msg->GetHeader()->Size = sizeof(TBusHeader) + compdata.Size(); @@ -721,21 +721,21 @@ namespace NBus { } else { WriteDummyHeader(*data); CallSerialize(msg, *data); - + dataSize = msg->GetHeader()->Size = data->Size() - pos; - + data->Proceed(pos); WriteHeader(*msg->GetHeader(), *data); data->Proceed(pos + msg->GetHeader()->Size); } - + Y_ASSERT(msg->GetHeader()->Size == data->Size() - pos); counter->AddMessage(dataSize, data->Size() - pos, msg->IsCompressed(), compressionRequested); } - + TBusMessage* TRemoteConnection::DeserializeMessage(TArrayRef<const char> dataRef, const TBusHeader* header, TMessageCounter* messageCounter, EMessageStatus* status) const { size_t dataSize; - + TBusMessage* message; if (header->FlagsInternal & MESSAGE_COMPRESS_INTERNAL) { TBuffer msg; @@ -751,7 +751,7 @@ namespace NBus { *status = MESSAGE_DECOMPRESS_ERROR; return nullptr; } - + msg.Append(dataRef.data(), sizeof(TBusHeader)); msg.Append(plaindata.Data(), plaindata.Size()); } @@ -774,39 +774,39 @@ namespace NBus { } *message->GetHeader() = *header; } - + messageCounter->AddMessage(dataSize, dataRef.size(), header->FlagsInternal & MESSAGE_COMPRESS_INTERNAL, false); - + return message; } - + void TRemoteConnection::ResetOneWayFlag(TArrayRef<TBusMessage*> messages) { for (auto message : messages) { message->LocalFlags &= ~MESSAGE_ONE_WAY_INTERNAL; } - } - + } + void TRemoteConnection::ReaderFlushMessages() { if (!ReaderData.ReadMessages.empty()) { Session->OnMessageReceived(this, ReaderData.ReadMessages); ReaderData.ReadMessages.clear(); - } - } - + } + } + // @return false if actor should break bool TRemoteConnection::MessageRead(TArrayRef<const char> readDataRef, TInstant now) { TBusHeader header(readDataRef); - + Y_ASSERT(readDataRef.size() == header.Size); - + if (header.GetVersionInternal() != YBUS_VERSION) { ReaderProcessMessageUnknownVersion(readDataRef); return true; } - + EMessageStatus deserializeFailureStatus = MESSAGE_OK; TBusMessage* r = DeserializeMessage(readDataRef, &header, &ReaderData.Status.Incremental.MessageCounter, &deserializeFailureStatus); - + if (!r) { Y_VERIFY(deserializeFailureStatus != MESSAGE_OK, "state check"); LWPROBE(Error, ToString(deserializeFailureStatus), ToString(PeerAddr), ""); @@ -814,16 +814,16 @@ namespace NBus { ScheduleShutdownOnServerOrReconnectOnClient(deserializeFailureStatus, false); return false; } - + LWPROBE(Read, r->GetHeader()->Size); - + r->ReplyTo = PeerAddrSocketAddr; - + TBusMessagePtrAndHeader h(r); r->RecvTime = now; QuotaConsume(1, header.Size); - + ReaderData.ReadMessages.push_back(h); if (ReaderData.ReadMessages.size() >= 100) { ReaderFlushMessages(); @@ -831,12 +831,12 @@ namespace NBus { return true; } - + void TRemoteConnection::WriterFillBuffer() { Y_ASSERT(WriterData.State == WRITER_FILLING); Y_ASSERT(WriterData.Buffer.LeftSize() == 0); - + if (Y_UNLIKELY(!WrongVersionRequests.IsEmpty())) { TVector<TBusHeader> headers; WrongVersionRequests.DequeueAllSingleConsumer(&headers); @@ -849,12 +849,12 @@ namespace NBus { response.SetVersionInternal(YBUS_VERSION); WriteHeader(response, WriterData.Buffer.GetBuffer()); } - + Y_ASSERT(!WriterData.Buffer.Empty()); WriterData.State = WRITER_FLUSHING; return; } - + TTempTlsVector<TBusMessagePtrAndHeader, void, TVectorSwaps> writeMessages; for (;;) { @@ -862,7 +862,7 @@ namespace NBus { if (!writeMessage) { break; } - + if (Config.Cork != TDuration::Zero()) { if (WriterData.CorkUntil == TInstant::Zero()) { WriterData.CorkUntil = TInstant::Now() + Config.Cork; @@ -870,29 +870,29 @@ namespace NBus { } size_t sizeBeforeSerialize = WriterData.Buffer.Size(); - + TMessageCounter messageCounter = WriterData.Status.Incremental.MessageCounter; - + SerializeMessage(writeMessage.Get(), &WriterData.Buffer.GetBuffer(), &messageCounter); - + size_t written = WriterData.Buffer.Size() - sizeBeforeSerialize; if (written > Config.MaxMessageSize) { WriterData.Buffer.GetBuffer().EraseBack(written); WriterBeforeWriteErrorMessage(writeMessage.Release(), MESSAGE_MESSAGE_TOO_LARGE); continue; } - + WriterData.Status.Incremental.MessageCounter = messageCounter; - + TBusMessagePtrAndHeader h(writeMessage.Release()); writeMessages.GetVector()->push_back(h); - + Y_ASSERT(!WriterData.Buffer.Empty()); if (WriterData.Buffer.Size() >= Config.SendThreshold) { break; } - } - + } + if (!WriterData.Buffer.Empty()) { if (WriterData.Buffer.Size() >= Config.SendThreshold) { WriterData.State = WRITER_FLUSHING; @@ -909,31 +909,31 @@ namespace NBus { // keep filling Y_ASSERT(WriterData.State == WRITER_FILLING); } - + size_t bytes = MessageSize(*writeMessages.GetVector()); - + QuotaReturnSelf(writeMessages.GetVector()->size(), bytes); - + // This is called before `send` syscall inducing latency MessageSent(*writeMessages.GetVector()); - } - + } + size_t TRemoteConnection::MessageSize(TArrayRef<TBusMessagePtrAndHeader> messages) { size_t size = 0; for (const auto& message : messages) size += message.MessagePtr->RequestSize; - + return size; } - + size_t TRemoteConnection::GetInFlight() { return AtomicGet(WriterData.InFlight); - } - + } + size_t TRemoteConnection::GetConnectSyscallsNumForTest() { return WriterData.Status.ConnectSyscalls; - } - + } + void TRemoteConnection::WriterBeforeWriteErrorMessage(TBusMessage* message, EMessageStatus status) { if (Session->IsSource_) { CheckedCast<TRemoteClientSession*>(Session.Get())->ReleaseInFlight({message}); @@ -970,5 +970,5 @@ namespace NBus { return !AtomicGet(WriterData.Down); } - } -} + } +} diff --git a/library/cpp/messagebus/remote_connection.h b/library/cpp/messagebus/remote_connection.h index 5481b2c6ed..4538947368 100644 --- a/library/cpp/messagebus/remote_connection.h +++ b/library/cpp/messagebus/remote_connection.h @@ -1,17 +1,17 @@ #pragma once -#include "async_result.h" +#include "async_result.h" #include "defs.h" #include "event_loop.h" #include "left_right_buffer.h" #include "lfqueue_batch.h" #include "message_ptr_and_header.h" -#include "nondestroying_holder.h" -#include "remote_connection_status.h" -#include "scheduler_actor.h" -#include "socket_addr.h" +#include "nondestroying_holder.h" +#include "remote_connection_status.h" +#include "scheduler_actor.h" +#include "socket_addr.h" #include "storage.h" -#include "vector_swaps.h" +#include "vector_swaps.h" #include "ybus.h" #include "misc/granup.h" #include "misc/tokenquota.h" @@ -29,23 +29,23 @@ namespace NBus { namespace NPrivate { class TRemoteConnection; - + typedef TIntrusivePtr<TRemoteConnection> TRemoteConnectionPtr; typedef TIntrusivePtr<TBusSessionImpl> TRemoteSessionPtr; - + static void* const WriteCookie = (void*)1; static void* const ReadCookie = (void*)2; - + enum { WAKE_QUOTA_MSG = 0x01, WAKE_QUOTA_BYTES = 0x02 }; - + struct TWriterTag {}; struct TReaderTag {}; struct TReconnectTag {}; struct TWakeReaderTag {}; - + struct TWriterToReaderSocketMessage { TSocket Socket; ui32 SocketVersion; @@ -56,7 +56,7 @@ namespace NBus { { } }; - + class TRemoteConnection : public NEventLoop::IEventHandler, public ::NActor::TActor<TRemoteConnection, TWriterTag>, @@ -71,43 +71,43 @@ namespace NBus { friend class ::NActor::TQueueInActor<TRemoteConnection, TWriterToReaderSocketMessage, TReaderTag>; friend class ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TReconnectTag>; friend class ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TWakeReaderTag>; - + protected: ::NActor::TQueueInActor<TRemoteConnection, TWriterToReaderSocketMessage, TReaderTag>* ReaderGetSocketQueue() { return this; } - + ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TReconnectTag>* WriterGetReconnectQueue() { return this; } - + ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TWakeReaderTag>* WriterGetWakeQueue() { return this; } - + protected: TRemoteConnection(TRemoteSessionPtr session, ui64 connectionId, TNetAddr addr); ~TRemoteConnection() override; - + virtual void ClearOutgoingQueue(TMessagesPtrs&, bool reconnect /* or shutdown */); - + public: void Send(TNonDestroyingAutoPtr<TBusMessage> msg); void Shutdown(EMessageStatus status); - + inline const TNetAddr& GetAddr() const noexcept; private: friend class TScheduleConnect; friend class TWorkIO; - + protected: static size_t MessageSize(TArrayRef<TBusMessagePtrAndHeader>); bool QuotaAcquire(size_t msg, size_t bytes); void QuotaConsume(size_t msg, size_t bytes); void QuotaReturnSelf(size_t items, size_t bytes); bool QuotaReturnValues(size_t items, size_t bytes); - + bool ReaderProcessBuffer(); bool ReaderFillBuffer(); void ReaderFlushMessages(); @@ -170,22 +170,22 @@ namespace NBus { inline TScheduleActor<TRemoteConnection, TWriterTag>* GetWriterSchedulerActor() { return this; } - + void WriterErrorMessage(TNonDestroyingAutoPtr<TBusMessage> m, EMessageStatus status); // takes ownership of ms void WriterErrorMessages(const TArrayRef<TBusMessage*> ms, EMessageStatus status); void FireClientConnectionEvent(TClientConnectionEvent::EType); - + size_t GetInFlight(); size_t GetConnectSyscallsNumForTest(); - + bool IsReturnConnectFailedImmediately() { return (bool)AtomicGet(ReturnConnectFailedImmediately); } - + bool IsAlive() const; - + TRemoteSessionPtr Session; TBusProtocol* const Proto; TBusSessionConfig const Config; @@ -193,15 +193,15 @@ namespace NBus { const ui64 ConnectionId; const TNetAddr PeerAddr; const TBusSocketAddr PeerAddrSocketAddr; - + const TInstant CreatedTime; TInstant LastConnectAttempt; TAtomic ReturnConnectFailedImmediately; - + protected: ::NActor::TQueueForActor<TBusMessage*> BeforeSendQueue; TLockFreeStack<TBusHeader> WrongVersionRequests; - + struct TWriterData { TAtomic Down; @@ -212,7 +212,7 @@ namespace NBus { TInstant StatusLastSendTime; TLocalTasks TimeToRotateCounters; - + TAtomic InFlight; TTimedMessages SendQueue; @@ -220,44 +220,44 @@ namespace NBus { EWriterState State; TLeftRightBuffer Buffer; TInstant CorkUntil; - + TSystemEvent ShutdownComplete; - + void SetChannel(NEventLoop::TChannelPtr channel); void DropChannel(); - + TWriterData(); ~TWriterData(); }; - + struct TReaderData { TAtomic Down; - + NEventLoop::TChannelPtr Channel; ui32 SocketVersion; - + TRemoteConnectionReaderStatus Status; TInstant StatusLastSendTime; - + TBuffer Buffer; size_t Offset; /* offset in read buffer */ size_t MoreBytes; /* more bytes required from socket */ TVectorSwaps<TBusMessagePtrAndHeader> ReadMessages; - + TSystemEvent ShutdownComplete; - + bool BufferMore() const noexcept { return MoreBytes > 0; } - + bool HasBytesInBuf(size_t bytes) noexcept; void SetChannel(NEventLoop::TChannelPtr channel); void DropChannel(); - + TReaderData(); ~TReaderData(); }; - + // owned by session status actor struct TGranStatus { TGranStatus(TDuration gran) @@ -265,7 +265,7 @@ namespace NBus { , Reader(gran) { } - + TGranUp<TRemoteConnectionWriterStatus> Writer; TGranUp<TRemoteConnectionReaderStatus> Reader; }; @@ -275,15 +275,15 @@ namespace NBus { TGranStatus GranStatus; TTokenQuota QuotaMsg; TTokenQuota QuotaBytes; - + size_t MaxBufferSize; - + // client connection only TLockFreeQueueBatch<TBusMessagePtrAndHeader, TVectorSwaps> ReplyQueue; EMessageStatus ShutdownReason; - }; - + }; + inline const TNetAddr& TRemoteConnection::GetAddr() const noexcept { return PeerAddr; } diff --git a/library/cpp/messagebus/remote_connection_status.cpp b/library/cpp/messagebus/remote_connection_status.cpp index 0a1706ae31..2c48b2a287 100644 --- a/library/cpp/messagebus/remote_connection_status.cpp +++ b/library/cpp/messagebus/remote_connection_status.cpp @@ -1,133 +1,133 @@ #include "remote_connection_status.h" - -#include "key_value_printer.h" - + +#include "key_value_printer.h" + #include <library/cpp/messagebus/monitoring/mon_proto.pb.h> #include <util/stream/format.h> #include <util/stream/output.h> #include <util/system/yassert.h> -using namespace NBus; -using namespace NBus::NPrivate; - -template <typename T> -static void Add(T& thiz, const T& that) { - thiz += that; -} - -template <typename T> -static void Max(T& thiz, const T& that) { - if (that > thiz) { - thiz = that; - } -} - -template <typename T> -static void AssertZero(T& thiz, const T& that) { +using namespace NBus; +using namespace NBus::NPrivate; + +template <typename T> +static void Add(T& thiz, const T& that) { + thiz += that; +} + +template <typename T> +static void Max(T& thiz, const T& that) { + if (that > thiz) { + thiz = that; + } +} + +template <typename T> +static void AssertZero(T& thiz, const T& that) { Y_ASSERT(thiz == T()); Y_UNUSED(that); -} - -TDurationCounter::TDurationCounter() - : DURATION_COUNTER_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) -{ -} - -TDuration TDurationCounter::AvgDuration() const { - if (Count == 0) { - return TDuration::Zero(); - } else { - return SumDuration / Count; - } -} - -TDurationCounter& TDurationCounter::operator+=(const TDurationCounter& that) { - DURATION_COUNTER_MAP(STRUCT_FIELD_ADD, ) - return *this; -} - +} + +TDurationCounter::TDurationCounter() + : DURATION_COUNTER_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) +{ +} + +TDuration TDurationCounter::AvgDuration() const { + if (Count == 0) { + return TDuration::Zero(); + } else { + return SumDuration / Count; + } +} + +TDurationCounter& TDurationCounter::operator+=(const TDurationCounter& that) { + DURATION_COUNTER_MAP(STRUCT_FIELD_ADD, ) + return *this; +} + TString TDurationCounter::ToString() const { - if (Count == 0) { - return "0"; - } else { - TStringStream ss; - ss << "avg: " << AvgDuration() << ", max: " << MaxDuration << ", count: " << Count; - return ss.Str(); - } -} - -TRemoteConnectionStatusBase::TRemoteConnectionStatusBase() - : REMOTE_CONNECTION_STATUS_BASE_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) -{ -} - + if (Count == 0) { + return "0"; + } else { + TStringStream ss; + ss << "avg: " << AvgDuration() << ", max: " << MaxDuration << ", count: " << Count; + return ss.Str(); + } +} + +TRemoteConnectionStatusBase::TRemoteConnectionStatusBase() + : REMOTE_CONNECTION_STATUS_BASE_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) +{ +} + TRemoteConnectionStatusBase& TRemoteConnectionStatusBase ::operator+=(const TRemoteConnectionStatusBase& that) { - REMOTE_CONNECTION_STATUS_BASE_MAP(STRUCT_FIELD_ADD, ) - return *this; -} - -TRemoteConnectionIncrementalStatusBase::TRemoteConnectionIncrementalStatusBase() - : REMOTE_CONNECTION_INCREMENTAL_STATUS_BASE_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) -{ -} - + REMOTE_CONNECTION_STATUS_BASE_MAP(STRUCT_FIELD_ADD, ) + return *this; +} + +TRemoteConnectionIncrementalStatusBase::TRemoteConnectionIncrementalStatusBase() + : REMOTE_CONNECTION_INCREMENTAL_STATUS_BASE_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) +{ +} + TRemoteConnectionIncrementalStatusBase& TRemoteConnectionIncrementalStatusBase::operator+=( const TRemoteConnectionIncrementalStatusBase& that) { - REMOTE_CONNECTION_INCREMENTAL_STATUS_BASE_MAP(STRUCT_FIELD_ADD, ) - return *this; -} - -TRemoteConnectionReaderIncrementalStatus::TRemoteConnectionReaderIncrementalStatus() - : REMOTE_CONNECTION_READER_INCREMENTAL_STATUS_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) -{ -} - -TRemoteConnectionReaderIncrementalStatus& TRemoteConnectionReaderIncrementalStatus::operator+=( + REMOTE_CONNECTION_INCREMENTAL_STATUS_BASE_MAP(STRUCT_FIELD_ADD, ) + return *this; +} + +TRemoteConnectionReaderIncrementalStatus::TRemoteConnectionReaderIncrementalStatus() + : REMOTE_CONNECTION_READER_INCREMENTAL_STATUS_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) +{ +} + +TRemoteConnectionReaderIncrementalStatus& TRemoteConnectionReaderIncrementalStatus::operator+=( const TRemoteConnectionReaderIncrementalStatus& that) { TRemoteConnectionIncrementalStatusBase::operator+=(that); - REMOTE_CONNECTION_READER_INCREMENTAL_STATUS_MAP(STRUCT_FIELD_ADD, ) - return *this; -} - -TRemoteConnectionReaderStatus::TRemoteConnectionReaderStatus() - : REMOTE_CONNECTION_READER_STATUS_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) -{ -} - -TRemoteConnectionReaderStatus& TRemoteConnectionReaderStatus::operator+=(const TRemoteConnectionReaderStatus& that) { + REMOTE_CONNECTION_READER_INCREMENTAL_STATUS_MAP(STRUCT_FIELD_ADD, ) + return *this; +} + +TRemoteConnectionReaderStatus::TRemoteConnectionReaderStatus() + : REMOTE_CONNECTION_READER_STATUS_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) +{ +} + +TRemoteConnectionReaderStatus& TRemoteConnectionReaderStatus::operator+=(const TRemoteConnectionReaderStatus& that) { TRemoteConnectionStatusBase::operator+=(that); - REMOTE_CONNECTION_READER_STATUS_MAP(STRUCT_FIELD_ADD, ) - return *this; -} - -TRemoteConnectionWriterIncrementalStatus::TRemoteConnectionWriterIncrementalStatus() - : REMOTE_CONNECTION_WRITER_INCREMENTAL_STATUS(STRUCT_FIELD_INIT_DEFAULT, COMMA) -{ -} - -TRemoteConnectionWriterIncrementalStatus& TRemoteConnectionWriterIncrementalStatus::operator+=( + REMOTE_CONNECTION_READER_STATUS_MAP(STRUCT_FIELD_ADD, ) + return *this; +} + +TRemoteConnectionWriterIncrementalStatus::TRemoteConnectionWriterIncrementalStatus() + : REMOTE_CONNECTION_WRITER_INCREMENTAL_STATUS(STRUCT_FIELD_INIT_DEFAULT, COMMA) +{ +} + +TRemoteConnectionWriterIncrementalStatus& TRemoteConnectionWriterIncrementalStatus::operator+=( const TRemoteConnectionWriterIncrementalStatus& that) { TRemoteConnectionIncrementalStatusBase::operator+=(that); - REMOTE_CONNECTION_WRITER_INCREMENTAL_STATUS(STRUCT_FIELD_ADD, ) - return *this; -} - -TRemoteConnectionWriterStatus::TRemoteConnectionWriterStatus() - : REMOTE_CONNECTION_WRITER_STATUS(STRUCT_FIELD_INIT_DEFAULT, COMMA) -{ -} - -TRemoteConnectionWriterStatus& TRemoteConnectionWriterStatus::operator+=(const TRemoteConnectionWriterStatus& that) { + REMOTE_CONNECTION_WRITER_INCREMENTAL_STATUS(STRUCT_FIELD_ADD, ) + return *this; +} + +TRemoteConnectionWriterStatus::TRemoteConnectionWriterStatus() + : REMOTE_CONNECTION_WRITER_STATUS(STRUCT_FIELD_INIT_DEFAULT, COMMA) +{ +} + +TRemoteConnectionWriterStatus& TRemoteConnectionWriterStatus::operator+=(const TRemoteConnectionWriterStatus& that) { TRemoteConnectionStatusBase::operator+=(that); - REMOTE_CONNECTION_WRITER_STATUS(STRUCT_FIELD_ADD, ) - return *this; -} - -size_t TRemoteConnectionWriterStatus::GetInFlight() const { - return SendQueueSize + AckMessagesSize; -} - + REMOTE_CONNECTION_WRITER_STATUS(STRUCT_FIELD_ADD, ) + return *this; +} + +size_t TRemoteConnectionWriterStatus::GetInFlight() const { + return SendQueueSize + AckMessagesSize; +} + TConnectionStatusMonRecord TRemoteConnectionStatus::GetStatusProtobuf() const { TConnectionStatusMonRecord status; @@ -155,31 +155,31 @@ TConnectionStatusMonRecord TRemoteConnectionStatus::GetStatusProtobuf() const { } TString TRemoteConnectionStatus::PrintToString() const { - TStringStream ss; - - TKeyValuePrinter p; - - if (!Summary) { - // TODO: print MyAddr too, but only if it is set - ss << WriterStatus.PeerAddr << " (" << WriterStatus.ConnectionId << ")" + TStringStream ss; + + TKeyValuePrinter p; + + if (!Summary) { + // TODO: print MyAddr too, but only if it is set + ss << WriterStatus.PeerAddr << " (" << WriterStatus.ConnectionId << ")" << ", writefd=" << WriterStatus.Fd << ", readfd=" << ReaderStatus.Fd << Endl; - if (WriterStatus.Connected) { - p.AddRow("connect time", WriterStatus.ConnectTime.ToString()); - p.AddRow("writer state", ToCString(WriterStatus.State)); - } else { - ss << "not connected"; - if (WriterStatus.ConnectError != 0) { - ss << ", last connect error: " << LastSystemErrorText(WriterStatus.ConnectError); - } - ss << Endl; - } - } - if (!Server) { - p.AddRow("connect syscalls", WriterStatus.ConnectSyscalls); - } - + if (WriterStatus.Connected) { + p.AddRow("connect time", WriterStatus.ConnectTime.ToString()); + p.AddRow("writer state", ToCString(WriterStatus.State)); + } else { + ss << "not connected"; + if (WriterStatus.ConnectError != 0) { + ss << ", last connect error: " << LastSystemErrorText(WriterStatus.ConnectError); + } + ss << Endl; + } + } + if (!Server) { + p.AddRow("connect syscalls", WriterStatus.ConnectSyscalls); + } + p.AddRow("send queue", LeftPad(WriterStatus.SendQueueSize, 6)); if (Server) { @@ -189,77 +189,77 @@ TString TRemoteConnectionStatus::PrintToString() const { p.AddRow("reader wakeups", LeftPad(WriterStatus.ReaderWakeups, 6)); } else { p.AddRow("ack messages", LeftPad(WriterStatus.AckMessagesSize, 6)); - } - - p.AddRow("written", WriterStatus.Incremental.MessageCounter.ToString(false)); - p.AddRow("read", ReaderStatus.Incremental.MessageCounter.ToString(true)); - - p.AddRow("write syscalls", LeftPad(WriterStatus.Incremental.NetworkOps, 12)); - p.AddRow("read syscalls", LeftPad(ReaderStatus.Incremental.NetworkOps, 12)); - - p.AddRow("write acts", LeftPad(WriterStatus.Acts, 12)); - p.AddRow("read acts", LeftPad(ReaderStatus.Acts, 12)); - - p.AddRow("write buffer cap", LeftPad(WriterStatus.BufferSize, 12)); - p.AddRow("read buffer cap", LeftPad(ReaderStatus.BufferSize, 12)); - - p.AddRow("write buffer drops", LeftPad(WriterStatus.Incremental.BufferDrops, 10)); - p.AddRow("read buffer drops", LeftPad(ReaderStatus.Incremental.BufferDrops, 10)); - - if (Server) { - p.AddRow("process dur", WriterStatus.DurationCounterPrev.ToString()); - } - - ss << p.PrintToString(); - - if (false && Server) { - ss << "time histogram:\n"; - ss << WriterStatus.Incremental.ProcessDurationHistogram.PrintToString(); - } - - TMessageStatusCounter sumStatusCounter; - sumStatusCounter += WriterStatus.Incremental.StatusCounter; - sumStatusCounter += ReaderStatus.Incremental.StatusCounter; - - ss << sumStatusCounter.PrintToString(); - - return ss.Str(); -} - -TRemoteConnectionStatus::TRemoteConnectionStatus() - : REMOTE_CONNECTION_STATUS_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) -{ -} - + } + + p.AddRow("written", WriterStatus.Incremental.MessageCounter.ToString(false)); + p.AddRow("read", ReaderStatus.Incremental.MessageCounter.ToString(true)); + + p.AddRow("write syscalls", LeftPad(WriterStatus.Incremental.NetworkOps, 12)); + p.AddRow("read syscalls", LeftPad(ReaderStatus.Incremental.NetworkOps, 12)); + + p.AddRow("write acts", LeftPad(WriterStatus.Acts, 12)); + p.AddRow("read acts", LeftPad(ReaderStatus.Acts, 12)); + + p.AddRow("write buffer cap", LeftPad(WriterStatus.BufferSize, 12)); + p.AddRow("read buffer cap", LeftPad(ReaderStatus.BufferSize, 12)); + + p.AddRow("write buffer drops", LeftPad(WriterStatus.Incremental.BufferDrops, 10)); + p.AddRow("read buffer drops", LeftPad(ReaderStatus.Incremental.BufferDrops, 10)); + + if (Server) { + p.AddRow("process dur", WriterStatus.DurationCounterPrev.ToString()); + } + + ss << p.PrintToString(); + + if (false && Server) { + ss << "time histogram:\n"; + ss << WriterStatus.Incremental.ProcessDurationHistogram.PrintToString(); + } + + TMessageStatusCounter sumStatusCounter; + sumStatusCounter += WriterStatus.Incremental.StatusCounter; + sumStatusCounter += ReaderStatus.Incremental.StatusCounter; + + ss << sumStatusCounter.PrintToString(); + + return ss.Str(); +} + +TRemoteConnectionStatus::TRemoteConnectionStatus() + : REMOTE_CONNECTION_STATUS_MAP(STRUCT_FIELD_INIT_DEFAULT, COMMA) +{ +} + TString TSessionDumpStatus::PrintToString() const { - if (Shutdown) { - return "shutdown"; - } - - TStringStream ss; - ss << Head; - if (ConnectionStatusSummary.Server) { - ss << "\n"; - ss << Acceptors; - } - ss << "\n"; - ss << "connections summary:" << Endl; - ss << ConnectionsSummary; - if (!!Connections) { - ss << "\n"; - ss << Connections; - } - ss << "\n"; - ss << Config.PrintToString(); - return ss.Str(); -} - + if (Shutdown) { + return "shutdown"; + } + + TStringStream ss; + ss << Head; + if (ConnectionStatusSummary.Server) { + ss << "\n"; + ss << Acceptors; + } + ss << "\n"; + ss << "connections summary:" << Endl; + ss << ConnectionsSummary; + if (!!Connections) { + ss << "\n"; + ss << Connections; + } + ss << "\n"; + ss << Config.PrintToString(); + return ss.Str(); +} + TString TBusMessageQueueStatus::PrintToString() const { - TStringStream ss; - ss << "work queue:\n"; - ss << ExecutorStatus.Status; - ss << "\n"; - ss << "queue config:\n"; - ss << Config.PrintToString(); - return ss.Str(); -} + TStringStream ss; + ss << "work queue:\n"; + ss << ExecutorStatus.Status; + ss << "\n"; + ss << "queue config:\n"; + ss << Config.PrintToString(); + return ss.Str(); +} diff --git a/library/cpp/messagebus/remote_connection_status.h b/library/cpp/messagebus/remote_connection_status.h index 0431d7eb32..5db10e51ea 100644 --- a/library/cpp/messagebus/remote_connection_status.h +++ b/library/cpp/messagebus/remote_connection_status.h @@ -1,12 +1,12 @@ -#pragma once - -#include "codegen.h" +#pragma once + +#include "codegen.h" #include "duration_histogram.h" #include "message_counter.h" -#include "message_status_counter.h" -#include "queue_config.h" -#include "session_config.h" - +#include "message_status_counter.h" +#include "queue_config.h" +#include "session_config.h" + #include <library/cpp/messagebus/actor/executor.h> #include <library/cpp/deprecated/enum_codegen/enum_codegen.h> @@ -17,40 +17,40 @@ namespace NBus { namespace NBus { namespace NPrivate { -#define WRITER_STATE_MAP(XX) \ +#define WRITER_STATE_MAP(XX) \ XX(WRITER_UNKNOWN) \ XX(WRITER_FILLING) \ XX(WRITER_FLUSHING) \ - /**/ - + /**/ + // TODO: move elsewhere enum EWriterState { WRITER_STATE_MAP(ENUM_VALUE_GEN_NO_VALUE) }; - + ENUM_TO_STRING(EWriterState, WRITER_STATE_MAP) - -#define STRUCT_FIELD_ADD(name, type, func) func(name, that.name); - + +#define STRUCT_FIELD_ADD(name, type, func) func(name, that.name); + template <typename T> void Reset(T& t) { t.~T(); new (&t) T(); } - + #define DURATION_COUNTER_MAP(XX, comma) \ XX(Count, unsigned, Add) \ comma \ XX(SumDuration, TDuration, Add) comma \ XX(MaxDuration, TDuration, Max) /**/ - + struct TDurationCounter { DURATION_COUNTER_MAP(STRUCT_FIELD_GEN, ) - + TDuration AvgDuration() const; - + TDurationCounter(); - + void AddDuration(TDuration d) { Count += 1; SumDuration += d; @@ -58,82 +58,82 @@ namespace NBus { MaxDuration = d; } } - + TDurationCounter& operator+=(const TDurationCounter&); - + TString ToString() const; }; - -#define REMOTE_CONNECTION_STATUS_BASE_MAP(XX, comma) \ + +#define REMOTE_CONNECTION_STATUS_BASE_MAP(XX, comma) \ XX(ConnectionId, ui64, AssertZero) \ comma \ XX(Fd, SOCKET, AssertZero) comma \ XX(Acts, ui64, Add) comma \ XX(BufferSize, ui64, Add) /**/ - + struct TRemoteConnectionStatusBase { REMOTE_CONNECTION_STATUS_BASE_MAP(STRUCT_FIELD_GEN, ) - + TRemoteConnectionStatusBase& operator+=(const TRemoteConnectionStatusBase&); - + TRemoteConnectionStatusBase(); }; - -#define REMOTE_CONNECTION_INCREMENTAL_STATUS_BASE_MAP(XX, comma) \ + +#define REMOTE_CONNECTION_INCREMENTAL_STATUS_BASE_MAP(XX, comma) \ XX(BufferDrops, unsigned, Add) \ comma \ XX(NetworkOps, unsigned, Add) /**/ - + struct TRemoteConnectionIncrementalStatusBase { REMOTE_CONNECTION_INCREMENTAL_STATUS_BASE_MAP(STRUCT_FIELD_GEN, ) - + TRemoteConnectionIncrementalStatusBase& operator+=(const TRemoteConnectionIncrementalStatusBase&); - + TRemoteConnectionIncrementalStatusBase(); }; - -#define REMOTE_CONNECTION_READER_INCREMENTAL_STATUS_MAP(XX, comma) \ + +#define REMOTE_CONNECTION_READER_INCREMENTAL_STATUS_MAP(XX, comma) \ XX(MessageCounter, TMessageCounter, Add) \ comma \ XX(StatusCounter, TMessageStatusCounter, Add) /**/ - + struct TRemoteConnectionReaderIncrementalStatus: public TRemoteConnectionIncrementalStatusBase { REMOTE_CONNECTION_READER_INCREMENTAL_STATUS_MAP(STRUCT_FIELD_GEN, ) - + TRemoteConnectionReaderIncrementalStatus& operator+=(const TRemoteConnectionReaderIncrementalStatus&); - + TRemoteConnectionReaderIncrementalStatus(); }; - -#define REMOTE_CONNECTION_READER_STATUS_MAP(XX, comma) \ + +#define REMOTE_CONNECTION_READER_STATUS_MAP(XX, comma) \ XX(QuotaMsg, size_t, Add) \ comma \ XX(QuotaBytes, size_t, Add) comma \ XX(QuotaExhausted, size_t, Add) comma \ XX(Incremental, TRemoteConnectionReaderIncrementalStatus, Add) /**/ - + struct TRemoteConnectionReaderStatus: public TRemoteConnectionStatusBase { REMOTE_CONNECTION_READER_STATUS_MAP(STRUCT_FIELD_GEN, ) - + TRemoteConnectionReaderStatus& operator+=(const TRemoteConnectionReaderStatus&); - + TRemoteConnectionReaderStatus(); }; - -#define REMOTE_CONNECTION_WRITER_INCREMENTAL_STATUS(XX, comma) \ + +#define REMOTE_CONNECTION_WRITER_INCREMENTAL_STATUS(XX, comma) \ XX(MessageCounter, TMessageCounter, Add) \ comma \ XX(StatusCounter, TMessageStatusCounter, Add) comma \ XX(ProcessDurationHistogram, TDurationHistogram, Add) /**/ - + struct TRemoteConnectionWriterIncrementalStatus: public TRemoteConnectionIncrementalStatusBase { REMOTE_CONNECTION_WRITER_INCREMENTAL_STATUS(STRUCT_FIELD_GEN, ) - + TRemoteConnectionWriterIncrementalStatus& operator+=(const TRemoteConnectionWriterIncrementalStatus&); - + TRemoteConnectionWriterIncrementalStatus(); }; - + #define REMOTE_CONNECTION_WRITER_STATUS(XX, comma) \ XX(Connected, bool, AssertZero) \ comma \ @@ -149,42 +149,42 @@ namespace NBus { XX(DurationCounterPrev, TDurationCounter, Add) comma /* server only */ \ XX(Incremental, TRemoteConnectionWriterIncrementalStatus, Add) comma \ XX(ReaderWakeups, size_t, Add) /**/ - + struct TRemoteConnectionWriterStatus: public TRemoteConnectionStatusBase { REMOTE_CONNECTION_WRITER_STATUS(STRUCT_FIELD_GEN, ) - + TRemoteConnectionWriterStatus(); - + TRemoteConnectionWriterStatus& operator+=(const TRemoteConnectionWriterStatus&); - + size_t GetInFlight() const; }; - -#define REMOTE_CONNECTION_STATUS_MAP(XX, comma) \ + +#define REMOTE_CONNECTION_STATUS_MAP(XX, comma) \ XX(Summary, bool) \ comma \ XX(Server, bool) /**/ - + struct TRemoteConnectionStatus { REMOTE_CONNECTION_STATUS_MAP(STRUCT_FIELD_GEN, ) - + TRemoteConnectionReaderStatus ReaderStatus; TRemoteConnectionWriterStatus WriterStatus; - + TRemoteConnectionStatus(); - + TString PrintToString() const; TConnectionStatusMonRecord GetStatusProtobuf() const; }; - + struct TBusSessionStatus { size_t InFlightCount; size_t InFlightSize; bool InputPaused; - + TBusSessionStatus(); }; - + struct TSessionDumpStatus { bool Shutdown; TString Head; @@ -194,20 +194,20 @@ namespace NBus { TBusSessionStatus Status; TRemoteConnectionStatus ConnectionStatusSummary; TBusSessionConfig Config; - + TSessionDumpStatus() : Shutdown(false) { } - + TString PrintToString() const; }; - + // without sessions struct TBusMessageQueueStatus { NActor::NPrivate::TExecutorStatus ExecutorStatus; TBusQueueConfig Config; - + TString PrintToString() const; }; } diff --git a/library/cpp/messagebus/remote_server_connection.cpp b/library/cpp/messagebus/remote_server_connection.cpp index 9af7fb2185..74be34ded9 100644 --- a/library/cpp/messagebus/remote_server_connection.cpp +++ b/library/cpp/messagebus/remote_server_connection.cpp @@ -1,73 +1,73 @@ -#include "remote_server_connection.h" - +#include "remote_server_connection.h" + #include "mb_lwtrace.h" -#include "remote_server_session.h" - +#include "remote_server_session.h" + #include <util/generic/cast.h> - -LWTRACE_USING(LWTRACE_MESSAGEBUS_PROVIDER) - -using namespace NBus; -using namespace NBus::NPrivate; - -TRemoteServerConnection::TRemoteServerConnection(TRemoteServerSessionPtr session, ui64 id, TNetAddr addr) - : TRemoteConnection(session.Get(), id, addr) -{ -} - + +LWTRACE_USING(LWTRACE_MESSAGEBUS_PROVIDER) + +using namespace NBus; +using namespace NBus::NPrivate; + +TRemoteServerConnection::TRemoteServerConnection(TRemoteServerSessionPtr session, ui64 id, TNetAddr addr) + : TRemoteConnection(session.Get(), id, addr) +{ +} + void TRemoteServerConnection::Init(SOCKET socket, TInstant now) { - WriterData.Status.ConnectTime = now; - WriterData.Status.Connected = true; - + WriterData.Status.ConnectTime = now; + WriterData.Status.Connected = true; + Y_VERIFY(socket != INVALID_SOCKET, "must be a valid socket"); - - TSocket readSocket(socket); - TSocket writeSocket = readSocket; - - // this must not be done in constructor, because if event loop is stopped, - // this is deleted - WriterData.SetChannel(Session->WriteEventLoop.Register(writeSocket, this, WriteCookie)); - WriterData.SocketVersion = 1; - - ReaderGetSocketQueue()->EnqueueAndSchedule(TWriterToReaderSocketMessage(readSocket, WriterData.SocketVersion)); -} - -TRemoteServerSession* TRemoteServerConnection::GetSession() { - return CheckedCast<TRemoteServerSession*>(Session.Get()); -} - -void TRemoteServerConnection::HandleEvent(SOCKET socket, void* cookie) { + + TSocket readSocket(socket); + TSocket writeSocket = readSocket; + + // this must not be done in constructor, because if event loop is stopped, + // this is deleted + WriterData.SetChannel(Session->WriteEventLoop.Register(writeSocket, this, WriteCookie)); + WriterData.SocketVersion = 1; + + ReaderGetSocketQueue()->EnqueueAndSchedule(TWriterToReaderSocketMessage(readSocket, WriterData.SocketVersion)); +} + +TRemoteServerSession* TRemoteServerConnection::GetSession() { + return CheckedCast<TRemoteServerSession*>(Session.Get()); +} + +void TRemoteServerConnection::HandleEvent(SOCKET socket, void* cookie) { Y_UNUSED(socket); Y_ASSERT(cookie == ReadCookie || cookie == WriteCookie); - if (cookie == ReadCookie) { - GetSession()->ServerOwnedMessages.Wait(); - ScheduleRead(); - } else { - ScheduleWrite(); - } -} - -bool TRemoteServerConnection::NeedInterruptRead() { - return !GetSession()->ServerOwnedMessages.TryWait(); -} - + if (cookie == ReadCookie) { + GetSession()->ServerOwnedMessages.Wait(); + ScheduleRead(); + } else { + ScheduleWrite(); + } +} + +bool TRemoteServerConnection::NeedInterruptRead() { + return !GetSession()->ServerOwnedMessages.TryWait(); +} + void TRemoteServerConnection::MessageSent(TArrayRef<TBusMessagePtrAndHeader> messages) { - TInstant now = TInstant::Now(); - - GetSession()->ReleaseInWorkResponses(messages); + TInstant now = TInstant::Now(); + + GetSession()->ReleaseInWorkResponses(messages); for (auto& message : messages) { TInstant recvTime = message.MessagePtr->RecvTime; GetSession()->ServerHandler->OnSent(message.MessagePtr.Release()); - TDuration d = now - recvTime; - WriterData.Status.DurationCounter.AddDuration(d); - WriterData.Status.Incremental.ProcessDurationHistogram.AddTime(d); - } -} - + TDuration d = now - recvTime; + WriterData.Status.DurationCounter.AddDuration(d); + WriterData.Status.Incremental.ProcessDurationHistogram.AddTime(d); + } +} + void TRemoteServerConnection::ReaderProcessMessageUnknownVersion(TArrayRef<const char> dataRef) { - TBusHeader header(dataRef); - // TODO: full version hex - LWPROBE(ServerUnknownVersion, ToString(PeerAddr), header.GetVersionInternal()); - WrongVersionRequests.Enqueue(header); - GetWriterActor()->Schedule(); -} + TBusHeader header(dataRef); + // TODO: full version hex + LWPROBE(ServerUnknownVersion, ToString(PeerAddr), header.GetVersionInternal()); + WrongVersionRequests.Enqueue(header); + GetWriterActor()->Schedule(); +} diff --git a/library/cpp/messagebus/remote_server_connection.h b/library/cpp/messagebus/remote_server_connection.h index d91e62e8a6..63d7f20646 100644 --- a/library/cpp/messagebus/remote_server_connection.h +++ b/library/cpp/messagebus/remote_server_connection.h @@ -1,32 +1,32 @@ -#pragma once - +#pragma once + #include "session_impl.h" #include <util/generic/object_counter.h> - + namespace NBus { namespace NPrivate { class TRemoteServerConnection: public TRemoteConnection { friend struct TBusSessionImpl; friend class TRemoteServerSession; - + TObjectCounter<TRemoteServerConnection> ObjectCounter; - + public: TRemoteServerConnection(TRemoteServerSessionPtr session, ui64 id, TNetAddr addr); - + void Init(SOCKET socket, TInstant now); - + inline TRemoteServerSession* GetSession(); - + void HandleEvent(SOCKET socket, void* cookie) override; - + bool NeedInterruptRead() override; - + void MessageSent(TArrayRef<TBusMessagePtrAndHeader> messages) override; - + void ReaderProcessMessageUnknownVersion(TArrayRef<const char> dataRef) override; }; - + } } diff --git a/library/cpp/messagebus/remote_server_session.cpp b/library/cpp/messagebus/remote_server_session.cpp index c43207cf8a..6abbf88a60 100644 --- a/library/cpp/messagebus/remote_server_session.cpp +++ b/library/cpp/messagebus/remote_server_session.cpp @@ -1,26 +1,26 @@ #include "remote_server_session.h" - + #include "remote_connection.h" #include "remote_server_connection.h" #include <library/cpp/messagebus/actor/temp_tls_vector.h> #include <util/generic/cast.h> -#include <util/stream/output.h> -#include <util/system/yassert.h> - +#include <util/stream/output.h> +#include <util/system/yassert.h> + #include <typeinfo> - -using namespace NActor; -using namespace NBus; -using namespace NBus::NPrivate; - -TRemoteServerSession::TRemoteServerSession(TBusMessageQueue* queue, + +using namespace NActor; +using namespace NBus; +using namespace NBus::NPrivate; + +TRemoteServerSession::TRemoteServerSession(TBusMessageQueue* queue, TBusProtocol* proto, IBusServerHandler* handler, const TBusServerSessionConfig& config, const TString& name) - : TBusSessionImpl(false, queue, proto, handler, config, name) - , ServerOwnedMessages(config.MaxInFlight, config.MaxInFlightBySize, "ServerOwnedMessages") - , ServerHandler(handler) + : TBusSessionImpl(false, queue, proto, handler, config, name) + , ServerOwnedMessages(config.MaxInFlight, config.MaxInFlightBySize, "ServerOwnedMessages") + , ServerHandler(handler) { if (config.PerConnectionMaxInFlightBySize > 0) { if (config.PerConnectionMaxInFlightBySize < config.MaxMessageSize) @@ -36,17 +36,17 @@ namespace NBus { TRemoteServerSession* RemoteServerSession; TBusMessagePtrAndHeader Request; TIntrusivePtr<TRemoteServerConnection> Connection; - + public: TInvokeOnMessage(TRemoteServerSession* session, TBusMessagePtrAndHeader& request, TIntrusivePtr<TRemoteServerConnection>& connection) : RemoteServerSession(session) { Y_ASSERT(!!connection); Connection.Swap(connection); - + Request.Swap(request); } - + void DoWork() override { THolder<TInvokeOnMessage> holder(this); RemoteServerSession->InvokeOnMessage(Request, Connection); @@ -54,153 +54,153 @@ namespace NBus { RemoteServerSession->JobCount.Decrement(); } }; - + } } - -void TRemoteServerSession::OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& messages) { - AcquireInWorkRequests(messages); - - bool executeInPool = Config.ExecuteOnMessageInWorkerPool; - - TTempTlsVector< ::IWorkItem*> workQueueTemp; - - if (executeInPool) { - workQueueTemp.GetVector()->reserve(messages.size()); - } - + +void TRemoteServerSession::OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& messages) { + AcquireInWorkRequests(messages); + + bool executeInPool = Config.ExecuteOnMessageInWorkerPool; + + TTempTlsVector< ::IWorkItem*> workQueueTemp; + + if (executeInPool) { + workQueueTemp.GetVector()->reserve(messages.size()); + } + for (auto& message : messages) { - // TODO: incref once - TIntrusivePtr<TRemoteServerConnection> connection(CheckedCast<TRemoteServerConnection*>(c)); - if (executeInPool) { + // TODO: incref once + TIntrusivePtr<TRemoteServerConnection> connection(CheckedCast<TRemoteServerConnection*>(c)); + if (executeInPool) { workQueueTemp.GetVector()->push_back(new TInvokeOnMessage(this, message, connection)); - } else { + } else { InvokeOnMessage(message, connection); - } - } - - if (executeInPool) { - JobCount.Add(workQueueTemp.GetVector()->size()); - Queue->EnqueueWork(*workQueueTemp.GetVector()); - } -} - -void TRemoteServerSession::InvokeOnMessage(TBusMessagePtrAndHeader& request, TIntrusivePtr<TRemoteServerConnection>& conn) { + } + } + + if (executeInPool) { + JobCount.Add(workQueueTemp.GetVector()->size()); + Queue->EnqueueWork(*workQueueTemp.GetVector()); + } +} + +void TRemoteServerSession::InvokeOnMessage(TBusMessagePtrAndHeader& request, TIntrusivePtr<TRemoteServerConnection>& conn) { if (Y_UNLIKELY(AtomicGet(Down))) { ReleaseInWorkRequests(*conn.Get(), request.MessagePtr.Get()); - InvokeOnError(request.MessagePtr.Release(), MESSAGE_SHUTDOWN); - } else { - TWhatThreadDoesPushPop pp("OnMessage"); - - TBusIdentity ident; - - ident.Connection.Swap(conn); - request.MessagePtr->GetIdentity(ident); - + InvokeOnError(request.MessagePtr.Release(), MESSAGE_SHUTDOWN); + } else { + TWhatThreadDoesPushPop pp("OnMessage"); + + TBusIdentity ident; + + ident.Connection.Swap(conn); + request.MessagePtr->GetIdentity(ident); + Y_ASSERT(request.MessagePtr->LocalFlags & MESSAGE_IN_WORK); - DoSwap(request.MessagePtr->LocalFlags, ident.LocalFlags); - - ident.RecvTime = request.MessagePtr->RecvTime; - -#ifndef NDEBUG + DoSwap(request.MessagePtr->LocalFlags, ident.LocalFlags); + + ident.RecvTime = request.MessagePtr->RecvTime; + +#ifndef NDEBUG auto& message = *request.MessagePtr; ident.SetMessageType(typeid(message)); -#endif - - TOnMessageContext context(request.MessagePtr.Release(), ident, this); - ServerHandler->OnMessage(context); - } -} - -EMessageStatus TRemoteServerSession::ForgetRequest(const TBusIdentity& ident) { - ReleaseInWork(const_cast<TBusIdentity&>(ident)); - - return MESSAGE_OK; -} - -EMessageStatus TRemoteServerSession::SendReply(const TBusIdentity& ident, TBusMessage* reply) { - reply->CheckClean(); - - ConvertInWork(const_cast<TBusIdentity&>(ident), reply); - - reply->RecvTime = ident.RecvTime; - - ident.Connection->Send(reply); - - return MESSAGE_OK; -} - +#endif + + TOnMessageContext context(request.MessagePtr.Release(), ident, this); + ServerHandler->OnMessage(context); + } +} + +EMessageStatus TRemoteServerSession::ForgetRequest(const TBusIdentity& ident) { + ReleaseInWork(const_cast<TBusIdentity&>(ident)); + + return MESSAGE_OK; +} + +EMessageStatus TRemoteServerSession::SendReply(const TBusIdentity& ident, TBusMessage* reply) { + reply->CheckClean(); + + ConvertInWork(const_cast<TBusIdentity&>(ident), reply); + + reply->RecvTime = ident.RecvTime; + + ident.Connection->Send(reply); + + return MESSAGE_OK; +} + int TRemoteServerSession::GetInFlight() const noexcept { - return ServerOwnedMessages.GetCurrentCount(); -} - -void TRemoteServerSession::FillStatus() { - TBusSessionImpl::FillStatus(); - - // TODO: weird - StatusData.Status.InFlightCount = ServerOwnedMessages.GetCurrentCount(); - StatusData.Status.InFlightSize = ServerOwnedMessages.GetCurrentSize(); - StatusData.Status.InputPaused = ServerOwnedMessages.IsLocked(); -} - + return ServerOwnedMessages.GetCurrentCount(); +} + +void TRemoteServerSession::FillStatus() { + TBusSessionImpl::FillStatus(); + + // TODO: weird + StatusData.Status.InFlightCount = ServerOwnedMessages.GetCurrentCount(); + StatusData.Status.InFlightSize = ServerOwnedMessages.GetCurrentSize(); + StatusData.Status.InputPaused = ServerOwnedMessages.IsLocked(); +} + void TRemoteServerSession::AcquireInWorkRequests(TArrayRef<const TBusMessagePtrAndHeader> messages) { - TAtomicBase size = 0; + TAtomicBase size = 0; for (auto message = messages.begin(); message != messages.end(); ++message) { Y_ASSERT(!(message->MessagePtr->LocalFlags & MESSAGE_IN_WORK)); - message->MessagePtr->LocalFlags |= MESSAGE_IN_WORK; - size += message->MessagePtr->GetHeader()->Size; - } - - ServerOwnedMessages.IncrementMultiple(messages.size(), size); -} - + message->MessagePtr->LocalFlags |= MESSAGE_IN_WORK; + size += message->MessagePtr->GetHeader()->Size; + } + + ServerOwnedMessages.IncrementMultiple(messages.size(), size); +} + void TRemoteServerSession::ReleaseInWorkResponses(TArrayRef<const TBusMessagePtrAndHeader> responses) { - TAtomicBase size = 0; + TAtomicBase size = 0; for (auto response = responses.begin(); response != responses.end(); ++response) { Y_ASSERT((response->MessagePtr->LocalFlags & MESSAGE_REPLY_IS_BEGING_SENT)); - response->MessagePtr->LocalFlags &= ~MESSAGE_REPLY_IS_BEGING_SENT; - size += response->MessagePtr->RequestSize; - } - - ServerOwnedMessages.ReleaseMultiple(responses.size(), size); -} - + response->MessagePtr->LocalFlags &= ~MESSAGE_REPLY_IS_BEGING_SENT; + size += response->MessagePtr->RequestSize; + } + + ServerOwnedMessages.ReleaseMultiple(responses.size(), size); +} + void TRemoteServerSession::ReleaseInWorkRequests(TRemoteConnection& con, TBusMessage* request) { Y_ASSERT((request->LocalFlags & MESSAGE_IN_WORK)); request->LocalFlags &= ~MESSAGE_IN_WORK; - + const size_t size = request->GetHeader()->Size; con.QuotaReturnAside(1, size); ServerOwnedMessages.ReleaseMultiple(1, size); -} - +} + void TRemoteServerSession::ReleaseInWork(TBusIdentity& ident) { ident.SetInWork(false); ident.Connection->QuotaReturnAside(1, ident.Size); - + ServerOwnedMessages.ReleaseMultiple(1, ident.Size); -} - -void TRemoteServerSession::ConvertInWork(TBusIdentity& req, TBusMessage* reply) { - reply->SetIdentity(req); - - req.SetInWork(false); +} + +void TRemoteServerSession::ConvertInWork(TBusIdentity& req, TBusMessage* reply) { + reply->SetIdentity(req); + + req.SetInWork(false); Y_ASSERT(!(reply->LocalFlags & MESSAGE_REPLY_IS_BEGING_SENT)); - reply->LocalFlags |= MESSAGE_REPLY_IS_BEGING_SENT; - reply->RequestSize = req.Size; -} - -void TRemoteServerSession::Shutdown() { - ServerOwnedMessages.Stop(); - TBusSessionImpl::Shutdown(); -} - -void TRemoteServerSession::PauseInput(bool pause) { - ServerOwnedMessages.PauseByUsed(pause); -} - -unsigned TRemoteServerSession::GetActualListenPort() { + reply->LocalFlags |= MESSAGE_REPLY_IS_BEGING_SENT; + reply->RequestSize = req.Size; +} + +void TRemoteServerSession::Shutdown() { + ServerOwnedMessages.Stop(); + TBusSessionImpl::Shutdown(); +} + +void TRemoteServerSession::PauseInput(bool pause) { + ServerOwnedMessages.PauseByUsed(pause); +} + +unsigned TRemoteServerSession::GetActualListenPort() { Y_VERIFY(Config.ListenPort > 0, "state check"); - return Config.ListenPort; -} + return Config.ListenPort; +} diff --git a/library/cpp/messagebus/remote_server_session.h b/library/cpp/messagebus/remote_server_session.h index a0fdf3e2e3..f5c266a7f7 100644 --- a/library/cpp/messagebus/remote_server_session.h +++ b/library/cpp/messagebus/remote_server_session.h @@ -1,13 +1,13 @@ #pragma once -#include "remote_server_session_semaphore.h" -#include "session_impl.h" +#include "remote_server_session_semaphore.h" +#include "session_impl.h" #ifdef _MSC_VER #pragma warning(push) #pragma warning(disable : 4250) // 'NBus::NPrivate::TRemoteClientSession' : inherits 'NBus::NPrivate::TBusSessionImpl::NBus::NPrivate::TBusSessionImpl::GetConfig' via dominance #endif - + namespace NBus { namespace NPrivate { class TRemoteServerSession: public TBusServerSession, public TBusSessionImpl { @@ -15,37 +15,37 @@ namespace NBus { private: TObjectCounter<TRemoteServerSession> ObjectCounter; - + TRemoteServerSessionSemaphore ServerOwnedMessages; IBusServerHandler* const ServerHandler; - + public: TRemoteServerSession(TBusMessageQueue* queue, TBusProtocol* proto, IBusServerHandler* handler, const TBusSessionConfig& config, const TString& name); - + void OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) override; void InvokeOnMessage(TBusMessagePtrAndHeader& request, TIntrusivePtr<TRemoteServerConnection>& conn); - + EMessageStatus SendReply(const TBusIdentity& ident, TBusMessage* pRep) override; - + EMessageStatus ForgetRequest(const TBusIdentity& ident) override; - + int GetInFlight() const noexcept override; void FillStatus() override; - + void Shutdown() override; - + void PauseInput(bool pause) override; unsigned GetActualListenPort() override; - + void AcquireInWorkRequests(TArrayRef<const TBusMessagePtrAndHeader> requests); void ReleaseInWorkResponses(TArrayRef<const TBusMessagePtrAndHeader> responses); void ReleaseInWorkRequests(TRemoteConnection&, TBusMessage*); void ReleaseInWork(TBusIdentity&); void ConvertInWork(TBusIdentity& req, TBusMessage* reply); }; - + #ifdef _MSC_VER #pragma warning(pop) #endif diff --git a/library/cpp/messagebus/remote_server_session_semaphore.cpp b/library/cpp/messagebus/remote_server_session_semaphore.cpp index 1db0a4e41d..6094a3586e 100644 --- a/library/cpp/messagebus/remote_server_session_semaphore.cpp +++ b/library/cpp/messagebus/remote_server_session_semaphore.cpp @@ -1,59 +1,59 @@ #include "remote_server_session_semaphore.h" #include <util/stream/output.h> -#include <util/system/yassert.h> - -using namespace NBus; -using namespace NBus::NPrivate; - -TRemoteServerSessionSemaphore::TRemoteServerSessionSemaphore( +#include <util/system/yassert.h> + +using namespace NBus; +using namespace NBus::NPrivate; + +TRemoteServerSessionSemaphore::TRemoteServerSessionSemaphore( TAtomicBase limitCount, TAtomicBase limitSize, const char* name) - : Name(name) - , LimitCount(limitCount) - , LimitSize(limitSize) - , CurrentCount(0) - , CurrentSize(0) - , PausedByUser(0) - , StopSignal(0) -{ + : Name(name) + , LimitCount(limitCount) + , LimitSize(limitSize) + , CurrentCount(0) + , CurrentSize(0) + , PausedByUser(0) + , StopSignal(0) +{ Y_VERIFY(limitCount > 0, "limit must be > 0"); Y_UNUSED(Name); -} - +} + TRemoteServerSessionSemaphore::~TRemoteServerSessionSemaphore() { Y_VERIFY(AtomicGet(CurrentCount) == 0); - // TODO: fix spider and enable + // TODO: fix spider and enable //Y_VERIFY(AtomicGet(CurrentSize) == 0); -} - -bool TRemoteServerSessionSemaphore::TryWait() { +} + +bool TRemoteServerSessionSemaphore::TryWait() { if (Y_UNLIKELY(AtomicGet(StopSignal))) - return true; - if (AtomicGet(PausedByUser)) - return false; - if (AtomicGet(CurrentCount) < LimitCount && (LimitSize < 0 || AtomicGet(CurrentSize) < LimitSize)) - return true; - return false; -} - -void TRemoteServerSessionSemaphore::IncrementMultiple(TAtomicBase count, TAtomicBase size) { - AtomicAdd(CurrentCount, count); - AtomicAdd(CurrentSize, size); - Updated(); -} - -void TRemoteServerSessionSemaphore::ReleaseMultiple(TAtomicBase count, TAtomicBase size) { - AtomicSub(CurrentCount, count); - AtomicSub(CurrentSize, size); - Updated(); -} - -void TRemoteServerSessionSemaphore::Stop() { - AtomicSet(StopSignal, 1); - Updated(); -} - -void TRemoteServerSessionSemaphore::PauseByUsed(bool pause) { - AtomicSet(PausedByUser, pause); - Updated(); -} + return true; + if (AtomicGet(PausedByUser)) + return false; + if (AtomicGet(CurrentCount) < LimitCount && (LimitSize < 0 || AtomicGet(CurrentSize) < LimitSize)) + return true; + return false; +} + +void TRemoteServerSessionSemaphore::IncrementMultiple(TAtomicBase count, TAtomicBase size) { + AtomicAdd(CurrentCount, count); + AtomicAdd(CurrentSize, size); + Updated(); +} + +void TRemoteServerSessionSemaphore::ReleaseMultiple(TAtomicBase count, TAtomicBase size) { + AtomicSub(CurrentCount, count); + AtomicSub(CurrentSize, size); + Updated(); +} + +void TRemoteServerSessionSemaphore::Stop() { + AtomicSet(StopSignal, 1); + Updated(); +} + +void TRemoteServerSessionSemaphore::PauseByUsed(bool pause) { + AtomicSet(PausedByUser, pause); + Updated(); +} diff --git a/library/cpp/messagebus/remote_server_session_semaphore.h b/library/cpp/messagebus/remote_server_session_semaphore.h index 453dd96d57..de714fd342 100644 --- a/library/cpp/messagebus/remote_server_session_semaphore.h +++ b/library/cpp/messagebus/remote_server_session_semaphore.h @@ -1,42 +1,42 @@ -#pragma once - +#pragma once + #include "cc_semaphore.h" -#include <util/generic/noncopyable.h> - +#include <util/generic/noncopyable.h> + namespace NBus { namespace NPrivate { class TRemoteServerSessionSemaphore: public TComplexConditionSemaphore<TRemoteServerSessionSemaphore> { private: const char* const Name; - + TAtomicBase const LimitCount; TAtomicBase const LimitSize; TAtomic CurrentCount; TAtomic CurrentSize; TAtomic PausedByUser; TAtomic StopSignal; - + public: TRemoteServerSessionSemaphore(TAtomicBase limitCount, TAtomicBase limitSize, const char* name = "unnamed"); ~TRemoteServerSessionSemaphore(); - + TAtomicBase GetCurrentCount() const { return AtomicGet(CurrentCount); } TAtomicBase GetCurrentSize() const { return AtomicGet(CurrentSize); } - + void IncrementMultiple(TAtomicBase count, TAtomicBase size); bool TryWait(); void ReleaseMultiple(TAtomicBase count, TAtomicBase size); void Stop(); void PauseByUsed(bool pause); - + private: void CheckNeedToUnlock(); }; - + } } diff --git a/library/cpp/messagebus/scheduler/scheduler.cpp b/library/cpp/messagebus/scheduler/scheduler.cpp index 8c966da86d..5a5fe52894 100644 --- a/library/cpp/messagebus/scheduler/scheduler.cpp +++ b/library/cpp/messagebus/scheduler/scheduler.cpp @@ -5,19 +5,19 @@ #include <util/generic/yexception.h> //#include "dummy_debugger.h" - -using namespace NBus; -using namespace NBus::NPrivate; + +using namespace NBus; +using namespace NBus::NPrivate; class TScheduleDeadlineCompare { public: bool operator()(const IScheduleItemAutoPtr& i1, const IScheduleItemAutoPtr& i2) const noexcept { - return i1->GetScheduleTime() > i2->GetScheduleTime(); + return i1->GetScheduleTime() > i2->GetScheduleTime(); } }; TScheduler::TScheduler() - : StopThread(false) + : StopThread(false) , Thread([&] { this->SchedulerThread(); }) { } @@ -32,13 +32,13 @@ size_t TScheduler::Size() const { } void TScheduler::Stop() { - { - TGuard<TLock> guard(Lock); + { + TGuard<TLock> guard(Lock); Y_VERIFY(!StopThread, "Scheduler already stopped"); - StopThread = true; - CondVar.Signal(); - } - Thread.Get(); + StopThread = true; + CondVar.Signal(); + } + Thread.Get(); if (!!NextItem) { NextItem.Destroy(); @@ -50,70 +50,70 @@ void TScheduler::Stop() { } void TScheduler::Schedule(TAutoPtr<IScheduleItem> i) { - TGuard<TLock> lock(Lock); - if (StopThread) + TGuard<TLock> lock(Lock); + if (StopThread) return; - - if (!!NextItem) { - if (i->GetScheduleTime() < NextItem->GetScheduleTime()) { - DoSwap(i, NextItem); - } + + if (!!NextItem) { + if (i->GetScheduleTime() < NextItem->GetScheduleTime()) { + DoSwap(i, NextItem); + } } - + Items.push_back(i); - PushHeap(Items.begin(), Items.end(), TScheduleDeadlineCompare()); - - FillNextItem(); - - CondVar.Signal(); + PushHeap(Items.begin(), Items.end(), TScheduleDeadlineCompare()); + + FillNextItem(); + + CondVar.Signal(); +} + +void TScheduler::FillNextItem() { + if (!NextItem && !Items.empty()) { + PopHeap(Items.begin(), Items.end(), TScheduleDeadlineCompare()); + NextItem = Items.back(); + Items.erase(Items.end() - 1); + } } -void TScheduler::FillNextItem() { - if (!NextItem && !Items.empty()) { - PopHeap(Items.begin(), Items.end(), TScheduleDeadlineCompare()); - NextItem = Items.back(); - Items.erase(Items.end() - 1); - } -} - void TScheduler::SchedulerThread() { - for (;;) { - IScheduleItemAutoPtr current; - + for (;;) { + IScheduleItemAutoPtr current; + { - TGuard<TLock> guard(Lock); - - if (StopThread) { - break; + TGuard<TLock> guard(Lock); + + if (StopThread) { + break; + } + + if (!!NextItem) { + CondVar.WaitD(Lock, NextItem->GetScheduleTime()); + } else { + CondVar.WaitI(Lock); } - - if (!!NextItem) { - CondVar.WaitD(Lock, NextItem->GetScheduleTime()); - } else { - CondVar.WaitI(Lock); + + if (StopThread) { + break; } - - if (StopThread) { - break; - } - - // signal comes if either scheduler is to be stopped of there's work to do + + // signal comes if either scheduler is to be stopped of there's work to do Y_VERIFY(!!NextItem, "state check"); - - if (TInstant::Now() < NextItem->GetScheduleTime()) { - // NextItem is updated since WaitD - continue; - } - + + if (TInstant::Now() < NextItem->GetScheduleTime()) { + // NextItem is updated since WaitD + continue; + } + current = NextItem.Release(); } - - current->Do(); - current.Destroy(); - - { - TGuard<TLock> guard(Lock); - FillNextItem(); - } + + current->Do(); + current.Destroy(); + + { + TGuard<TLock> guard(Lock); + FillNextItem(); + } } } diff --git a/library/cpp/messagebus/scheduler/scheduler.h b/library/cpp/messagebus/scheduler/scheduler.h index 6114c3cc88..afcc0de55d 100644 --- a/library/cpp/messagebus/scheduler/scheduler.h +++ b/library/cpp/messagebus/scheduler/scheduler.h @@ -8,7 +8,7 @@ #include <util/generic/vector.h> #include <util/system/atomic.h> #include <util/system/condvar.h> -#include <util/system/mutex.h> +#include <util/system/mutex.h> #include <util/system/thread.h> namespace NBus { @@ -18,10 +18,10 @@ namespace NBus { inline IScheduleItem(TInstant scheduleTime) noexcept; virtual ~IScheduleItem() { } - + virtual void Do() = 0; inline TInstant GetScheduleTime() const noexcept; - + private: TInstant ScheduleTime; }; @@ -50,16 +50,16 @@ namespace NBus { TCondVar CondVar; TObjectCounter<TScheduler> ObjectCounter; - + bool StopThread; NThreading::TLegacyFuture<> Thread; }; - + inline IScheduleItem::IScheduleItem(TInstant scheduleTime) noexcept : ScheduleTime(scheduleTime) { } - + inline TInstant IScheduleItem::GetScheduleTime() const noexcept { return ScheduleTime; } diff --git a/library/cpp/messagebus/scheduler/scheduler_ut.cpp b/library/cpp/messagebus/scheduler/scheduler_ut.cpp index 35fcccdd29..a5ea641c10 100644 --- a/library/cpp/messagebus/scheduler/scheduler_ut.cpp +++ b/library/cpp/messagebus/scheduler/scheduler_ut.cpp @@ -1,36 +1,36 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "scheduler.h" - + +#include "scheduler.h" + #include <library/cpp/messagebus/misc/test_sync.h> - -using namespace NBus; -using namespace NBus::NPrivate; - + +using namespace NBus; +using namespace NBus::NPrivate; + Y_UNIT_TEST_SUITE(TSchedulerTests) { - struct TSimpleScheduleItem: public IScheduleItem { - TTestSync* const TestSync; - - TSimpleScheduleItem(TTestSync* testSync) - : IScheduleItem((TInstant::Now() + TDuration::MilliSeconds(1))) - , TestSync(testSync) + struct TSimpleScheduleItem: public IScheduleItem { + TTestSync* const TestSync; + + TSimpleScheduleItem(TTestSync* testSync) + : IScheduleItem((TInstant::Now() + TDuration::MilliSeconds(1))) + , TestSync(testSync) { } - + void Do() override { - TestSync->WaitForAndIncrement(0); - } - }; - + TestSync->WaitForAndIncrement(0); + } + }; + Y_UNIT_TEST(Simple) { - TTestSync testSync; - - TScheduler scheduler; - - scheduler.Schedule(new TSimpleScheduleItem(&testSync)); - - testSync.WaitForAndIncrement(1); - - scheduler.Stop(); - } -} + TTestSync testSync; + + TScheduler scheduler; + + scheduler.Schedule(new TSimpleScheduleItem(&testSync)); + + testSync.WaitForAndIncrement(1); + + scheduler.Stop(); + } +} diff --git a/library/cpp/messagebus/scheduler/ya.make b/library/cpp/messagebus/scheduler/ya.make index 382804c408..dcb7408a20 100644 --- a/library/cpp/messagebus/scheduler/ya.make +++ b/library/cpp/messagebus/scheduler/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - +LIBRARY() + OWNER(g:messagebus) - + PEERDIR( library/cpp/threading/future ) -SRCS( - scheduler.cpp -) - -END() +SRCS( + scheduler.cpp +) + +END() diff --git a/library/cpp/messagebus/scheduler_actor.h b/library/cpp/messagebus/scheduler_actor.h index ba9595249b..d0c23c94c4 100644 --- a/library/cpp/messagebus/scheduler_actor.h +++ b/library/cpp/messagebus/scheduler_actor.h @@ -1,51 +1,51 @@ -#pragma once - -#include "local_tasks.h" - +#pragma once + +#include "local_tasks.h" + #include <library/cpp/messagebus/actor/actor.h> #include <library/cpp/messagebus/actor/what_thread_does_guard.h> #include <library/cpp/messagebus/scheduler/scheduler.h> - + #include <util/system/mutex.h> - + namespace NBus { namespace NPrivate { template <typename TThis, typename TTag = NActor::TDefaultTag> class TScheduleActor { typedef NActor::TActor<TThis, TTag> TActorForMe; - + private: TScheduler* const Scheduler; - + TMutex Mutex; - + TInstant ScheduleTime; - + public: TLocalTasks Alarm; - + private: struct TScheduleItemImpl: public IScheduleItem { TIntrusivePtr<TThis> Thiz; - + TScheduleItemImpl(TIntrusivePtr<TThis> thiz, TInstant when) : IScheduleItem(when) , Thiz(thiz) { } - + void Do() override { { TWhatThreadDoesAcquireGuard<TMutex> guard(Thiz->Mutex, "scheduler actor: acquiring lock for Do"); - + if (Thiz->ScheduleTime == TInstant::Max()) { // was already fired return; } - + Thiz->ScheduleTime = TInstant::Max(); } - + Thiz->Alarm.AddTask(); Thiz->GetActorForMe()->Schedule(); } @@ -55,31 +55,31 @@ namespace NBus { TScheduleActor(TScheduler* scheduler) : Scheduler(scheduler) , ScheduleTime(TInstant::Max()) - { + { } - + /// call Act(TTag) at specified time, unless it is already scheduled at earlier time. void ScheduleAt(TInstant when) { TWhatThreadDoesAcquireGuard<TMutex> guard(Mutex, "scheduler: acquiring lock for ScheduleAt"); if (when > ScheduleTime) { // already scheduled - return; - } - + return; + } + ScheduleTime = when; Scheduler->Schedule(new TScheduleItemImpl(GetThis(), when)); - } - + } + private: TThis* GetThis() { return static_cast<TThis*>(this); } - + TActorForMe* GetActorForMe() { return static_cast<TActorForMe*>(GetThis()); } }; - - } + + } } diff --git a/library/cpp/messagebus/scheduler_actor_ut.cpp b/library/cpp/messagebus/scheduler_actor_ut.cpp index 63e4d58782..e81ffd3186 100644 --- a/library/cpp/messagebus/scheduler_actor_ut.cpp +++ b/library/cpp/messagebus/scheduler_actor_ut.cpp @@ -1,48 +1,48 @@ #include <library/cpp/testing/unittest/registar.h> - + #include "scheduler_actor.h" -#include "misc/test_sync.h" - -using namespace NBus; -using namespace NBus::NPrivate; -using namespace NActor; - +#include "misc/test_sync.h" + +using namespace NBus; +using namespace NBus::NPrivate; +using namespace NActor; + Y_UNIT_TEST_SUITE(TSchedulerActorTests) { struct TMyActor: public TAtomicRefCount<TMyActor>, public TActor<TMyActor>, public TScheduleActor<TMyActor> { - TTestSync TestSync; - - TMyActor(TExecutor* executor, TScheduler* scheduler) - : TActor<TMyActor>(executor) - , TScheduleActor<TMyActor>(scheduler) - , Iteration(0) + TTestSync TestSync; + + TMyActor(TExecutor* executor, TScheduler* scheduler) + : TActor<TMyActor>(executor) + , TScheduleActor<TMyActor>(scheduler) + , Iteration(0) { } - - unsigned Iteration; - - void Act(TDefaultTag) { - if (!Alarm.FetchTask()) { + + unsigned Iteration; + + void Act(TDefaultTag) { + if (!Alarm.FetchTask()) { Y_FAIL("must not have no spurious wakeups in test"); - } - - TestSync.WaitForAndIncrement(Iteration++); - if (Iteration <= 5) { - ScheduleAt(TInstant::Now() + TDuration::MilliSeconds(Iteration)); - } - } - }; - + } + + TestSync.WaitForAndIncrement(Iteration++); + if (Iteration <= 5) { + ScheduleAt(TInstant::Now() + TDuration::MilliSeconds(Iteration)); + } + } + }; + Y_UNIT_TEST(Simple) { - TExecutor executor(1); - TScheduler scheduler; - - TIntrusivePtr<TMyActor> actor(new TMyActor(&executor, &scheduler)); - - actor->ScheduleAt(TInstant::Now() + TDuration::MilliSeconds(1)); - - actor->TestSync.WaitForAndIncrement(6); - - // TODO: stop in destructor - scheduler.Stop(); - } -} + TExecutor executor(1); + TScheduler scheduler; + + TIntrusivePtr<TMyActor> actor(new TMyActor(&executor, &scheduler)); + + actor->ScheduleAt(TInstant::Now() + TDuration::MilliSeconds(1)); + + actor->TestSync.WaitForAndIncrement(6); + + // TODO: stop in destructor + scheduler.Stop(); + } +} diff --git a/library/cpp/messagebus/session.cpp b/library/cpp/messagebus/session.cpp index 57d7ead401..46a7ece6a8 100644 --- a/library/cpp/messagebus/session.cpp +++ b/library/cpp/messagebus/session.cpp @@ -1,16 +1,16 @@ #include "ybus.h" -#include <util/generic/cast.h> - -using namespace NBus; - +#include <util/generic/cast.h> + +using namespace NBus; + namespace NBus { TBusSession::TBusSession() { } //////////////////////////////////////////////////////////////////// /// \brief Adds peer of connection into connection list - + int CompareByHost(const IRemoteAddr& l, const IRemoteAddr& r) noexcept { if (l.Addr()->sa_family != r.Addr()->sa_family) { return l.Addr()->sa_family < r.Addr()->sa_family ? -1 : +1; @@ -50,7 +50,7 @@ namespace NBus { bool SplitHost(const TString& host, TString* hostName, TString* portNum) { hostName->clear(); portNum->clear(); - + // Simple check that we have to deal with ipv6 address specification or // just host name or ipv4 address. if (!host.empty() && (host[0] == '[')) { @@ -97,7 +97,7 @@ namespace NBus { if (!SplitHost(host, &hostName, &port)) { hostName = host; } - + if (port.empty()) { portNum = GetProto()->GetPort(); } else { @@ -118,12 +118,12 @@ namespace NBus { } TBusClientSessionPtr TBusClientSession::Create(TBusProtocol* proto, IBusClientHandler* handler, const TBusClientSessionConfig& config, TBusMessageQueuePtr queue) { - return queue->CreateSource(proto, handler, config); -} - + return queue->CreateSource(proto, handler, config); +} + TBusServerSessionPtr TBusServerSession::Create(TBusProtocol* proto, IBusServerHandler* handler, const TBusServerSessionConfig& config, TBusMessageQueuePtr queue) { - return queue->CreateDestination(proto, handler, config); -} + return queue->CreateDestination(proto, handler, config); +} TBusServerSessionPtr TBusServerSession::Create(TBusProtocol* proto, IBusServerHandler* handler, const TBusServerSessionConfig& config, TBusMessageQueuePtr queue, const TVector<TBindResult>& bindTo) { return queue->CreateDestination(proto, handler, config, bindTo); diff --git a/library/cpp/messagebus/session.h b/library/cpp/messagebus/session.h index 5a1a01d808..fb12ab7c22 100644 --- a/library/cpp/messagebus/session.h +++ b/library/cpp/messagebus/session.h @@ -1,41 +1,41 @@ -#pragma once - +#pragma once + #include "connection.h" -#include "defs.h" +#include "defs.h" #include "handler.h" -#include "message.h" -#include "netaddr.h" +#include "message.h" +#include "netaddr.h" #include "network.h" -#include "session_config.h" +#include "session_config.h" #include "misc/weak_ptr.h" - + #include <library/cpp/messagebus/monitoring/mon_proto.pb.h> - + #include <util/generic/array_ref.h> #include <util/generic/ptr.h> -namespace NBus { +namespace NBus { template <typename TBusSessionSubclass> class TBusSessionPtr; using TBusClientSessionPtr = TBusSessionPtr<TBusClientSession>; using TBusServerSessionPtr = TBusSessionPtr<TBusServerSession>; - + /////////////////////////////////////////////////////////////////// /// \brief Interface of session object. - + /// Each client and server /// should instantiate session object to be able to communicate via bus /// client: sess = queue->CreateSource(protocol, handler); /// server: sess = queue->CreateDestination(protocol, handler); - + class TBusSession: public TWeakRefCounted<TBusSession> { public: size_t GetInFlight(const TNetAddr& addr) const; size_t GetConnectSyscallsNumForTest(const TNetAddr& addr) const; - + virtual void GetInFlightBulk(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const = 0; virtual void GetConnectSyscallsNumBulkForTest(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const = 0; - + virtual int GetInFlight() const noexcept = 0; /// monitoring status of current session and it's connections virtual TString GetStatus(ui16 flags = YBUS_STATUS_CONNS) = 0; @@ -47,42 +47,42 @@ namespace NBus { /// return session protocol virtual const TBusProtocol* GetProto() const noexcept = 0; virtual TBusMessageQueue* GetQueue() const noexcept = 0; - + /// registers external session on host:port with locator service int RegisterService(const char* hostname, TBusKey start = YBUS_KEYMIN, TBusKey end = YBUS_KEYMAX, EIpVersion ipVersion = EIP_VERSION_4); - + protected: TBusSession(); - + public: virtual TString GetNameInternal() = 0; - + virtual void Shutdown() = 0; - + virtual ~TBusSession(); }; - + struct TBusClientSession: public virtual TBusSession { typedef ::NBus::NPrivate::TRemoteClientSession TImpl; - + static TBusClientSessionPtr Create( TBusProtocol* proto, IBusClientHandler* handler, - const TBusClientSessionConfig& config, - TBusMessageQueuePtr queue); - + const TBusClientSessionConfig& config, + TBusMessageQueuePtr queue); + virtual TBusClientConnectionPtr GetConnection(const TNetAddr&) = 0; - + /// if you want to open connection early virtual void OpenConnection(const TNetAddr&) = 0; - + /// Send message to the destination /// If addr is set then use it as destination. /// Takes ownership of addr (see ClearState method). virtual EMessageStatus SendMessage(TBusMessage* pMes, const TNetAddr* addr = nullptr, bool wait = false) = 0; - + virtual EMessageStatus SendMessageOneWay(TBusMessage* pMes, const TNetAddr* addr = nullptr, bool wait = false) = 0; - + /// Like SendMessage but cares about message template <typename T /* <: TBusMessage */> EMessageStatus SendMessageAutoPtr(const TAutoPtr<T>& mes, const TNetAddr* addr = nullptr, bool wait = false) { @@ -91,7 +91,7 @@ namespace NBus { Y_UNUSED(mes.Release()); return status; } - + /// Like SendMessageOneWay but cares about message template <typename T /* <: TBusMessage */> EMessageStatus SendMessageOneWayAutoPtr(const TAutoPtr<T>& mes, const TNetAddr* addr = nullptr, bool wait = false) { @@ -100,27 +100,27 @@ namespace NBus { Y_UNUSED(mes.Release()); return status; } - + EMessageStatus SendMessageMove(TBusMessageAutoPtr message, const TNetAddr* addr = nullptr, bool wait = false) { return SendMessageAutoPtr(message, addr, wait); } - + EMessageStatus SendMessageOneWayMove(TBusMessageAutoPtr message, const TNetAddr* addr = nullptr, bool wait = false) { return SendMessageOneWayAutoPtr(message, addr, wait); } - + // TODO: implement similar one-way methods }; - + struct TBusServerSession: public virtual TBusSession { typedef ::NBus::NPrivate::TRemoteServerSession TImpl; - + static TBusServerSessionPtr Create( TBusProtocol* proto, IBusServerHandler* handler, - const TBusServerSessionConfig& config, - TBusMessageQueuePtr queue); - + const TBusServerSessionConfig& config, + TBusMessageQueuePtr queue); + static TBusServerSessionPtr Create( TBusProtocol* proto, IBusServerHandler* handler, @@ -130,10 +130,10 @@ namespace NBus { // TODO: make parameter non-const virtual EMessageStatus SendReply(const TBusIdentity& ident, TBusMessage* pRep) = 0; - + // TODO: make parameter non-const virtual EMessageStatus ForgetRequest(const TBusIdentity& ident) = 0; - + template <typename U /* <: TBusMessage */> EMessageStatus SendReplyAutoPtr(TBusIdentity& ident, TAutoPtr<U>& resp) { EMessageStatus status = SendReply(const_cast<const TBusIdentity&>(ident), resp.Get()); @@ -141,49 +141,49 @@ namespace NBus { Y_UNUSED(resp.Release()); } return status; - } - + } + EMessageStatus SendReplyMove(TBusIdentity& ident, TBusMessageAutoPtr resp) { return SendReplyAutoPtr(ident, resp); } - + /// Pause input from the network. /// It is valid to call this method in parallel. /// TODO: pull this method up to TBusSession. virtual void PauseInput(bool pause) = 0; virtual unsigned GetActualListenPort() = 0; }; - + namespace NPrivate { template <typename TBusSessionSubclass> class TBusOwnerSessionPtr: public TAtomicRefCount<TBusOwnerSessionPtr<TBusSessionSubclass>> { private: TIntrusivePtr<TBusSessionSubclass> Ptr; - + public: TBusOwnerSessionPtr(TBusSessionSubclass* session) : Ptr(session) { Y_ASSERT(!!Ptr); } - + ~TBusOwnerSessionPtr() { Ptr->Shutdown(); } - + TBusSessionSubclass* Get() const { return reinterpret_cast<TBusSessionSubclass*>(Ptr.Get()); } }; - - } - + + } + template <typename TBusSessionSubclass> class TBusSessionPtr { private: TIntrusivePtr<NPrivate::TBusOwnerSessionPtr<TBusSessionSubclass>> SmartPtr; TBusSessionSubclass* Ptr; - + public: TBusSessionPtr() : Ptr() @@ -194,7 +194,7 @@ namespace NBus { , Ptr(session) { } - + TBusSessionSubclass* Get() const { return Ptr; } @@ -207,19 +207,19 @@ namespace NBus { TBusSessionSubclass* operator->() const { return Get(); } - + bool operator!() const { return !Ptr; } - + void Swap(TBusSessionPtr& t) noexcept { DoSwap(SmartPtr, t.SmartPtr); DoSwap(Ptr, t.Ptr); } - + void Drop() { TBusSessionPtr().Swap(*this); } }; - -} + +} diff --git a/library/cpp/messagebus/session_config.h b/library/cpp/messagebus/session_config.h index ac0dce4291..37df97e986 100644 --- a/library/cpp/messagebus/session_config.h +++ b/library/cpp/messagebus/session_config.h @@ -1,4 +1,4 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/config/session_config.h> - + diff --git a/library/cpp/messagebus/session_impl.cpp b/library/cpp/messagebus/session_impl.cpp index fd123f7dd5..ddf9f360c4 100644 --- a/library/cpp/messagebus/session_impl.cpp +++ b/library/cpp/messagebus/session_impl.cpp @@ -1,216 +1,216 @@ #include "session_impl.h" - + #include "acceptor.h" -#include "network.h" +#include "network.h" #include "remote_client_connection.h" -#include "remote_client_session.h" -#include "remote_server_connection.h" +#include "remote_client_session.h" +#include "remote_server_connection.h" #include "remote_server_session.h" #include "misc/weak_ptr.h" - + #include <util/generic/cast.h> -using namespace NActor; -using namespace NBus; -using namespace NBus::NPrivate; -using namespace NEventLoop; - -namespace { - class TScheduleSession: public IScheduleItem { - public: - TScheduleSession(TBusSessionImpl* session, TInstant deadline) - : IScheduleItem(deadline) - , Session(session) - , SessionImpl(session) - { - } - +using namespace NActor; +using namespace NBus; +using namespace NBus::NPrivate; +using namespace NEventLoop; + +namespace { + class TScheduleSession: public IScheduleItem { + public: + TScheduleSession(TBusSessionImpl* session, TInstant deadline) + : IScheduleItem(deadline) + , Session(session) + , SessionImpl(session) + { + } + void Do() override { - TIntrusivePtr<TBusSession> session = Session.Get(); - if (!!session) { - SessionImpl->Cron(); - } - } - - private: - TWeakPtr<TBusSession> Session; - // Work around TWeakPtr limitation - TBusSessionImpl* SessionImpl; - }; -} - -TConnectionsAcceptorsSnapshot::TConnectionsAcceptorsSnapshot() + TIntrusivePtr<TBusSession> session = Session.Get(); + if (!!session) { + SessionImpl->Cron(); + } + } + + private: + TWeakPtr<TBusSession> Session; + // Work around TWeakPtr limitation + TBusSessionImpl* SessionImpl; + }; +} + +TConnectionsAcceptorsSnapshot::TConnectionsAcceptorsSnapshot() : LastConnectionId(0) , LastAcceptorId(0) { } - -struct TBusSessionImpl::TImpl { - TRemoteConnectionWriterIncrementalStatus DeadConnectionWriterStatusSummary; - TRemoteConnectionReaderIncrementalStatus DeadConnectionReaderStatusSummary; - TAcceptorStatus DeadAcceptorStatusSummary; -}; - -namespace { + +struct TBusSessionImpl::TImpl { + TRemoteConnectionWriterIncrementalStatus DeadConnectionWriterStatusSummary; + TRemoteConnectionReaderIncrementalStatus DeadConnectionReaderStatusSummary; + TAcceptorStatus DeadAcceptorStatusSummary; +}; + +namespace { TBusSessionConfig SessionConfigFillDefaults(const TBusSessionConfig& config, const TString& name) { - TBusSessionConfig copy = config; - if (copy.TotalTimeout == 0 && copy.SendTimeout == 0) { - copy.TotalTimeout = TDuration::Seconds(60).MilliSeconds(); - copy.SendTimeout = TDuration::Seconds(15).MilliSeconds(); - } else if (copy.TotalTimeout == 0) { + TBusSessionConfig copy = config; + if (copy.TotalTimeout == 0 && copy.SendTimeout == 0) { + copy.TotalTimeout = TDuration::Seconds(60).MilliSeconds(); + copy.SendTimeout = TDuration::Seconds(15).MilliSeconds(); + } else if (copy.TotalTimeout == 0) { Y_ASSERT(copy.SendTimeout != 0); - copy.TotalTimeout = config.SendTimeout + TDuration::MilliSeconds(10).MilliSeconds(); - } else if (copy.SendTimeout == 0) { + copy.TotalTimeout = config.SendTimeout + TDuration::MilliSeconds(10).MilliSeconds(); + } else if (copy.SendTimeout == 0) { Y_ASSERT(copy.TotalTimeout != 0); if ((ui64)copy.TotalTimeout > (ui64)TDuration::MilliSeconds(10).MilliSeconds()) { - copy.SendTimeout = copy.TotalTimeout - TDuration::MilliSeconds(10).MilliSeconds(); - } else { - copy.SendTimeout = copy.TotalTimeout; - } - } else { + copy.SendTimeout = copy.TotalTimeout - TDuration::MilliSeconds(10).MilliSeconds(); + } else { + copy.SendTimeout = copy.TotalTimeout; + } + } else { Y_ASSERT(copy.TotalTimeout != 0); Y_ASSERT(copy.SendTimeout != 0); - } - - if (copy.ConnectTimeout == 0) { - copy.ConnectTimeout = copy.SendTimeout; - } - + } + + if (copy.ConnectTimeout == 0) { + copy.ConnectTimeout = copy.SendTimeout; + } + Y_VERIFY(copy.SendTimeout > 0, "SendTimeout must be > 0"); Y_VERIFY(copy.TotalTimeout > 0, "TotalTimeout must be > 0"); Y_VERIFY(copy.ConnectTimeout > 0, "ConnectTimeout must be > 0"); Y_VERIFY(copy.TotalTimeout >= copy.SendTimeout, "TotalTimeout must be >= SendTimeout"); - - if (!copy.Name) { - copy.Name = name; - } - - return copy; - } -} - -TBusSessionImpl::TBusSessionImpl(bool isSource, TBusMessageQueue* queue, TBusProtocol* proto, + + if (!copy.Name) { + copy.Name = name; + } + + return copy; + } +} + +TBusSessionImpl::TBusSessionImpl(bool isSource, TBusMessageQueue* queue, TBusProtocol* proto, IBusErrorHandler* handler, const TBusSessionConfig& config, const TString& name) - : TActor<TBusSessionImpl, TStatusTag>(queue->WorkQueue.Get()) - , TActor<TBusSessionImpl, TConnectionTag>(queue->WorkQueue.Get()) - , Impl(new TImpl) + : TActor<TBusSessionImpl, TStatusTag>(queue->WorkQueue.Get()) + , TActor<TBusSessionImpl, TConnectionTag>(queue->WorkQueue.Get()) + , Impl(new TImpl) , IsSource_(isSource) - , Queue(queue) - , Proto(proto) - , ProtoName(Proto->GetService()) - , ErrorHandler(handler) - , HandlerUseCountHolder(&handler->UseCountChecker) - , Config(SessionConfigFillDefaults(config, name)) - , WriteEventLoop("wr-el") - , ReadEventLoop("rd-el") - , LastAcceptorId(0) - , LastConnectionId(0) + , Queue(queue) + , Proto(proto) + , ProtoName(Proto->GetService()) + , ErrorHandler(handler) + , HandlerUseCountHolder(&handler->UseCountChecker) + , Config(SessionConfigFillDefaults(config, name)) + , WriteEventLoop("wr-el") + , ReadEventLoop("rd-el") + , LastAcceptorId(0) + , LastConnectionId(0) , Down(0) { - Impl->DeadAcceptorStatusSummary.Summary = true; - + Impl->DeadAcceptorStatusSummary.Summary = true; + ReadEventLoopThread.Reset(new NThreading::TLegacyFuture<void, false>(std::bind(&TEventLoop::Run, std::ref(ReadEventLoop)))); WriteEventLoopThread.Reset(new NThreading::TLegacyFuture<void, false>(std::bind(&TEventLoop::Run, std::ref(WriteEventLoop)))); - - Queue->Schedule(IScheduleItemAutoPtr(new TScheduleSession(this, TInstant::Now() + Config.Secret.TimeoutPeriod))); + + Queue->Schedule(IScheduleItemAutoPtr(new TScheduleSession(this, TInstant::Now() + Config.Secret.TimeoutPeriod))); } -TBusSessionImpl::~TBusSessionImpl() { +TBusSessionImpl::~TBusSessionImpl() { Y_VERIFY(Down); Y_VERIFY(ShutdownCompleteEvent.WaitT(TDuration::Zero())); Y_VERIFY(!WriteEventLoop.IsRunning()); Y_VERIFY(!ReadEventLoop.IsRunning()); -} - -TBusSessionStatus::TBusSessionStatus() - : InFlightCount(0) - , InFlightSize(0) - , InputPaused(false) +} + +TBusSessionStatus::TBusSessionStatus() + : InFlightCount(0) + , InFlightSize(0) + , InputPaused(false) { } - -void TBusSessionImpl::Shutdown() { - if (!AtomicCas(&Down, 1, 0)) { - ShutdownCompleteEvent.WaitI(); - return; - } - + +void TBusSessionImpl::Shutdown() { + if (!AtomicCas(&Down, 1, 0)) { + ShutdownCompleteEvent.WaitI(); + return; + } + Y_VERIFY(Queue->IsRunning(), "Session must be shut down prior to queue shutdown"); - - TUseAfterFreeCheckerGuard handlerAliveCheckedGuard(ErrorHandler->UseAfterFreeChecker); - - // For legacy clients that don't use smart pointers - TIntrusivePtr<TBusSessionImpl> thiz(this); - - Queue->Remove(this); - - // shutdown event loops first, so they won't send more events - // to acceptors and connections - ReadEventLoop.Stop(); - WriteEventLoop.Stop(); - ReadEventLoopThread->Get(); - WriteEventLoopThread->Get(); - - // shutdown acceptors before connections - // so they won't create more connections + + TUseAfterFreeCheckerGuard handlerAliveCheckedGuard(ErrorHandler->UseAfterFreeChecker); + + // For legacy clients that don't use smart pointers + TIntrusivePtr<TBusSessionImpl> thiz(this); + + Queue->Remove(this); + + // shutdown event loops first, so they won't send more events + // to acceptors and connections + ReadEventLoop.Stop(); + WriteEventLoop.Stop(); + ReadEventLoopThread->Get(); + WriteEventLoopThread->Get(); + + // shutdown acceptors before connections + // so they won't create more connections TVector<TAcceptorPtr> acceptors; - GetAcceptors(&acceptors); - { - TGuard<TMutex> guard(ConnectionsLock); - Acceptors.clear(); - } + GetAcceptors(&acceptors); + { + TGuard<TMutex> guard(ConnectionsLock); + Acceptors.clear(); + } for (auto& acceptor : acceptors) { acceptor->Shutdown(); } - // shutdown connections + // shutdown connections TVector<TRemoteConnectionPtr> cs; - GetConnections(&cs); - + GetConnections(&cs); + for (auto& c : cs) { c->Shutdown(MESSAGE_SHUTDOWN); - } - - // shutdown connections actor - // must shutdown after connections destroyed - ConnectionsData.ShutdownState.ShutdownCommand(); - GetConnectionsActor()->Schedule(); - ConnectionsData.ShutdownState.ShutdownComplete.WaitI(); - - // finally shutdown status actor - StatusData.ShutdownState.ShutdownCommand(); - GetStatusActor()->Schedule(); - StatusData.ShutdownState.ShutdownComplete.WaitI(); - - // Make sure no one references IMessageHandler after Shutdown() - JobCount.WaitForZero(); - HandlerUseCountHolder.Reset(); - - ShutdownCompleteEvent.Signal(); -} - -bool TBusSessionImpl::IsDown() { + } + + // shutdown connections actor + // must shutdown after connections destroyed + ConnectionsData.ShutdownState.ShutdownCommand(); + GetConnectionsActor()->Schedule(); + ConnectionsData.ShutdownState.ShutdownComplete.WaitI(); + + // finally shutdown status actor + StatusData.ShutdownState.ShutdownCommand(); + GetStatusActor()->Schedule(); + StatusData.ShutdownState.ShutdownComplete.WaitI(); + + // Make sure no one references IMessageHandler after Shutdown() + JobCount.WaitForZero(); + HandlerUseCountHolder.Reset(); + + ShutdownCompleteEvent.Signal(); +} + +bool TBusSessionImpl::IsDown() { return static_cast<bool>(AtomicGet(Down)); } -size_t TBusSessionImpl::GetInFlightImpl(const TNetAddr& addr) const { - TRemoteConnectionPtr conn = const_cast<TBusSessionImpl*>(this)->GetConnection(addr, false); - if (!!conn) { - return conn->GetInFlight(); - } else { - return 0; - } +size_t TBusSessionImpl::GetInFlightImpl(const TNetAddr& addr) const { + TRemoteConnectionPtr conn = const_cast<TBusSessionImpl*>(this)->GetConnection(addr, false); + if (!!conn) { + return conn->GetInFlight(); + } else { + return 0; + } } void TBusSessionImpl::GetInFlightBulk(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const { Y_VERIFY(addrs.size() == results.size(), "input.size != output.size"); - for (size_t i = 0; i < addrs.size(); ++i) { - results[i] = GetInFlightImpl(addrs[i]); - } -} - + for (size_t i = 0; i < addrs.size(); ++i) { + results[i] = GetInFlightImpl(addrs[i]); + } +} + size_t TBusSessionImpl::GetConnectSyscallsNumForTestImpl(const TNetAddr& addr) const { TRemoteConnectionPtr conn = const_cast<TBusSessionImpl*>(this)->GetConnection(addr, false); if (!!conn) { @@ -227,26 +227,26 @@ void TBusSessionImpl::GetConnectSyscallsNumBulkForTest(TArrayRef<const TNetAddr> } } -void TBusSessionImpl::FillStatus() { -} - -TSessionDumpStatus TBusSessionImpl::GetStatusRecordInternal() { - // Probably useless, because it returns cached info now +void TBusSessionImpl::FillStatus() { +} + +TSessionDumpStatus TBusSessionImpl::GetStatusRecordInternal() { + // Probably useless, because it returns cached info now Y_VERIFY(!Queue->GetExecutor()->IsInExecutorThread(), "GetStatus must not be called from executor thread"); - - TGuard<TMutex> guard(StatusData.StatusDumpCachedMutex); - // TODO: returns zeros for a second after start - // (until first cron) - return StatusData.StatusDumpCached; -} - + + TGuard<TMutex> guard(StatusData.StatusDumpCachedMutex); + // TODO: returns zeros for a second after start + // (until first cron) + return StatusData.StatusDumpCached; +} + TString TBusSessionImpl::GetStatus(ui16 flags) { Y_UNUSED(flags); - - return GetStatusRecordInternal().PrintToString(); -} - + + return GetStatusRecordInternal().PrintToString(); +} + TConnectionStatusMonRecord TBusSessionImpl::GetStatusProtobuf() { Y_VERIFY(!Queue->GetExecutor()->IsInExecutorThread(), "GetStatus must not be called from executor thread"); @@ -257,233 +257,233 @@ TConnectionStatusMonRecord TBusSessionImpl::GetStatusProtobuf() { } TString TBusSessionImpl::GetStatusSingleLine() { - TSessionDumpStatus status = GetStatusRecordInternal(); - - TStringStream ss; - ss << "in-flight: " << status.Status.InFlightCount; - if (IsSource_) { - ss << " ack: " << status.ConnectionStatusSummary.WriterStatus.AckMessagesSize; - } - ss << " send-q: " << status.ConnectionStatusSummary.WriterStatus.SendQueueSize; - return ss.Str(); -} - -void TBusSessionImpl::ProcessItem(TStatusTag, TDeadConnectionTag, const TRemoteConnectionWriterIncrementalStatus& connectionStatus) { - Impl->DeadConnectionWriterStatusSummary += connectionStatus; -} - -void TBusSessionImpl::ProcessItem(TStatusTag, TDeadConnectionTag, const TRemoteConnectionReaderIncrementalStatus& connectionStatus) { - Impl->DeadConnectionReaderStatusSummary += connectionStatus; -} - -void TBusSessionImpl::ProcessItem(TStatusTag, TDeadConnectionTag, const TAcceptorStatus& acceptorStatus) { - Impl->DeadAcceptorStatusSummary += acceptorStatus; -} - -void TBusSessionImpl::ProcessItem(TConnectionTag, ::NActor::TDefaultTag, const TOnAccept& onAccept) { - TSocketHolder socket(onAccept.s); - - if (AtomicGet(Down)) { - // do not create connections after shutdown initiated - return; - } - - //if (Connections.find(addr) != Connections.end()) { + TSessionDumpStatus status = GetStatusRecordInternal(); + + TStringStream ss; + ss << "in-flight: " << status.Status.InFlightCount; + if (IsSource_) { + ss << " ack: " << status.ConnectionStatusSummary.WriterStatus.AckMessagesSize; + } + ss << " send-q: " << status.ConnectionStatusSummary.WriterStatus.SendQueueSize; + return ss.Str(); +} + +void TBusSessionImpl::ProcessItem(TStatusTag, TDeadConnectionTag, const TRemoteConnectionWriterIncrementalStatus& connectionStatus) { + Impl->DeadConnectionWriterStatusSummary += connectionStatus; +} + +void TBusSessionImpl::ProcessItem(TStatusTag, TDeadConnectionTag, const TRemoteConnectionReaderIncrementalStatus& connectionStatus) { + Impl->DeadConnectionReaderStatusSummary += connectionStatus; +} + +void TBusSessionImpl::ProcessItem(TStatusTag, TDeadConnectionTag, const TAcceptorStatus& acceptorStatus) { + Impl->DeadAcceptorStatusSummary += acceptorStatus; +} + +void TBusSessionImpl::ProcessItem(TConnectionTag, ::NActor::TDefaultTag, const TOnAccept& onAccept) { + TSocketHolder socket(onAccept.s); + + if (AtomicGet(Down)) { + // do not create connections after shutdown initiated + return; + } + + //if (Connections.find(addr) != Connections.end()) { // TODO: it is possible // won't be a problem after socket address replaced with id - //} - - TRemoteConnectionPtr c(new TRemoteServerConnection(VerifyDynamicCast<TRemoteServerSession*>(this), ++LastConnectionId, onAccept.addr)); - - VerifyDynamicCast<TRemoteServerConnection*>(c.Get())->Init(socket.Release(), onAccept.now); - - InsertConnectionLockAcquired(c.Get()); -} - -void TBusSessionImpl::ProcessItem(TConnectionTag, TRemoveTag, TRemoteConnectionPtr c) { - TAddrRemoteConnections::iterator it1 = Connections.find(c->PeerAddrSocketAddr); - if (it1 != Connections.end()) { - if (it1->second.Get() == c.Get()) { - Connections.erase(it1); - } - } - + //} + + TRemoteConnectionPtr c(new TRemoteServerConnection(VerifyDynamicCast<TRemoteServerSession*>(this), ++LastConnectionId, onAccept.addr)); + + VerifyDynamicCast<TRemoteServerConnection*>(c.Get())->Init(socket.Release(), onAccept.now); + + InsertConnectionLockAcquired(c.Get()); +} + +void TBusSessionImpl::ProcessItem(TConnectionTag, TRemoveTag, TRemoteConnectionPtr c) { + TAddrRemoteConnections::iterator it1 = Connections.find(c->PeerAddrSocketAddr); + if (it1 != Connections.end()) { + if (it1->second.Get() == c.Get()) { + Connections.erase(it1); + } + } + THashMap<ui64, TRemoteConnectionPtr>::iterator it2 = ConnectionsById.find(c->ConnectionId); - if (it2 != ConnectionsById.end()) { - ConnectionsById.erase(it2); - } - - SendSnapshotToStatusActor(); -} - + if (it2 != ConnectionsById.end()) { + ConnectionsById.erase(it2); + } + + SendSnapshotToStatusActor(); +} + void TBusSessionImpl::ProcessConnectionsAcceptorsShapshotQueueItem(TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> snapshot) { for (TVector<TRemoteConnectionPtr>::const_iterator connection = snapshot->Connections.begin(); connection != snapshot->Connections.end(); ++connection) { Y_ASSERT((*connection)->ConnectionId <= snapshot->LastConnectionId); - } - + } + for (TVector<TAcceptorPtr>::const_iterator acceptor = snapshot->Acceptors.begin(); acceptor != snapshot->Acceptors.end(); ++acceptor) { Y_ASSERT((*acceptor)->AcceptorId <= snapshot->LastAcceptorId); - } - - StatusData.ConnectionsAcceptorsSnapshot = snapshot; -} - -void TBusSessionImpl::StatusUpdateCachedDumpIfNecessary(TInstant now) { - if (now - StatusData.StatusDumpCachedLastUpdate > Config.Secret.StatusFlushPeriod) { - StatusUpdateCachedDump(); - StatusData.StatusDumpCachedLastUpdate = now; - } -} - -void TBusSessionImpl::StatusUpdateCachedDump() { - TSessionDumpStatus r; - - if (AtomicGet(Down)) { - r.Shutdown = true; - TGuard<TMutex> guard(StatusData.StatusDumpCachedMutex); - StatusData.StatusDumpCached = r; - return; - } - - // TODO: make thread-safe - FillStatus(); - - r.Status = StatusData.Status; - - { - TStringStream ss; - + } + + StatusData.ConnectionsAcceptorsSnapshot = snapshot; +} + +void TBusSessionImpl::StatusUpdateCachedDumpIfNecessary(TInstant now) { + if (now - StatusData.StatusDumpCachedLastUpdate > Config.Secret.StatusFlushPeriod) { + StatusUpdateCachedDump(); + StatusData.StatusDumpCachedLastUpdate = now; + } +} + +void TBusSessionImpl::StatusUpdateCachedDump() { + TSessionDumpStatus r; + + if (AtomicGet(Down)) { + r.Shutdown = true; + TGuard<TMutex> guard(StatusData.StatusDumpCachedMutex); + StatusData.StatusDumpCached = r; + return; + } + + // TODO: make thread-safe + FillStatus(); + + r.Status = StatusData.Status; + + { + TStringStream ss; + TString name = Config.Name; - if (!name) { - name = "unnamed"; - } - - ss << (IsSource_ ? "client" : "server") << " session " << name << ", proto " << Proto->GetService() << Endl; - ss << "in flight: " << r.Status.InFlightCount; - if (!IsSource_) { - ss << ", " << r.Status.InFlightSize << "b"; - } - if (r.Status.InputPaused) { - ss << " (input paused)"; - } - ss << "\n"; - - r.Head = ss.Str(); - } - + if (!name) { + name = "unnamed"; + } + + ss << (IsSource_ ? "client" : "server") << " session " << name << ", proto " << Proto->GetService() << Endl; + ss << "in flight: " << r.Status.InFlightCount; + if (!IsSource_) { + ss << ", " << r.Status.InFlightSize << "b"; + } + if (r.Status.InputPaused) { + ss << " (input paused)"; + } + ss << "\n"; + + r.Head = ss.Str(); + } + TVector<TRemoteConnectionPtr>& connections = StatusData.ConnectionsAcceptorsSnapshot->Connections; TVector<TAcceptorPtr>& acceptors = StatusData.ConnectionsAcceptorsSnapshot->Acceptors; - - r.ConnectionStatusSummary = TRemoteConnectionStatus(); - r.ConnectionStatusSummary.Summary = true; - r.ConnectionStatusSummary.Server = !IsSource_; - r.ConnectionStatusSummary.WriterStatus.Incremental = Impl->DeadConnectionWriterStatusSummary; - r.ConnectionStatusSummary.ReaderStatus.Incremental = Impl->DeadConnectionReaderStatusSummary; - - TAcceptorStatus acceptorStatusSummary = Impl->DeadAcceptorStatusSummary; - - { - TStringStream ss; - + + r.ConnectionStatusSummary = TRemoteConnectionStatus(); + r.ConnectionStatusSummary.Summary = true; + r.ConnectionStatusSummary.Server = !IsSource_; + r.ConnectionStatusSummary.WriterStatus.Incremental = Impl->DeadConnectionWriterStatusSummary; + r.ConnectionStatusSummary.ReaderStatus.Incremental = Impl->DeadConnectionReaderStatusSummary; + + TAcceptorStatus acceptorStatusSummary = Impl->DeadAcceptorStatusSummary; + + { + TStringStream ss; + for (TVector<TAcceptorPtr>::const_iterator acceptor = acceptors.begin(); acceptor != acceptors.end(); ++acceptor) { const TAcceptorStatus status = (*acceptor)->GranStatus.Listen.Get(); acceptorStatusSummary += status; - if (acceptor != acceptors.begin()) { - ss << "\n"; - } + if (acceptor != acceptors.begin()) { + ss << "\n"; + } ss << status.PrintToString(); - } - - r.Acceptors = ss.Str(); - } - - { - TStringStream ss; - + } + + r.Acceptors = ss.Str(); + } + + { + TStringStream ss; + for (TVector<TRemoteConnectionPtr>::const_iterator connection = connections.begin(); connection != connections.end(); ++connection) { - if (connection != connections.begin()) { - ss << "\n"; - } + if (connection != connections.begin()) { + ss << "\n"; + } - TRemoteConnectionStatus status; - status.Server = !IsSource_; + TRemoteConnectionStatus status; + status.Server = !IsSource_; status.ReaderStatus = (*connection)->GranStatus.Reader.Get(); status.WriterStatus = (*connection)->GranStatus.Writer.Get(); - ss << status.PrintToString(); + ss << status.PrintToString(); r.ConnectionStatusSummary.ReaderStatus += status.ReaderStatus; r.ConnectionStatusSummary.WriterStatus += status.WriterStatus; - } - + } + r.ConnectionsSummary = r.ConnectionStatusSummary.PrintToString(); - r.Connections = ss.Str(); - } - - r.Config = Config; - - TGuard<TMutex> guard(StatusData.StatusDumpCachedMutex); - StatusData.StatusDumpCached = r; -} - -TBusSessionImpl::TStatusData::TStatusData() - : ConnectionsAcceptorsSnapshot(new TConnectionsAcceptorsSnapshot) + r.Connections = ss.Str(); + } + + r.Config = Config; + + TGuard<TMutex> guard(StatusData.StatusDumpCachedMutex); + StatusData.StatusDumpCached = r; +} + +TBusSessionImpl::TStatusData::TStatusData() + : ConnectionsAcceptorsSnapshot(new TConnectionsAcceptorsSnapshot) { } - -void TBusSessionImpl::Act(TStatusTag) { - TInstant now = TInstant::Now(); - - EShutdownState shutdownState = StatusData.ShutdownState.State.Get(); - + +void TBusSessionImpl::Act(TStatusTag) { + TInstant now = TInstant::Now(); + + EShutdownState shutdownState = StatusData.ShutdownState.State.Get(); + StatusData.ConnectionsAcceptorsSnapshotsQueue.DequeueAllLikelyEmpty(std::bind(&TBusSessionImpl::ProcessConnectionsAcceptorsShapshotQueueItem, this, std::placeholders::_1)); - - GetDeadConnectionWriterStatusQueue()->DequeueAllLikelyEmpty(); - GetDeadConnectionReaderStatusQueue()->DequeueAllLikelyEmpty(); - GetDeadAcceptorStatusQueue()->DequeueAllLikelyEmpty(); - - // TODO: check queues are empty if already stopped - - if (shutdownState != SS_RUNNING) { - // important to beak cyclic link session -> connection -> session - StatusData.ConnectionsAcceptorsSnapshot->Connections.clear(); - StatusData.ConnectionsAcceptorsSnapshot->Acceptors.clear(); - } - - if (shutdownState == SS_SHUTDOWN_COMMAND) { - StatusData.ShutdownState.CompleteShutdown(); - } - - StatusUpdateCachedDumpIfNecessary(now); -} - + + GetDeadConnectionWriterStatusQueue()->DequeueAllLikelyEmpty(); + GetDeadConnectionReaderStatusQueue()->DequeueAllLikelyEmpty(); + GetDeadAcceptorStatusQueue()->DequeueAllLikelyEmpty(); + + // TODO: check queues are empty if already stopped + + if (shutdownState != SS_RUNNING) { + // important to beak cyclic link session -> connection -> session + StatusData.ConnectionsAcceptorsSnapshot->Connections.clear(); + StatusData.ConnectionsAcceptorsSnapshot->Acceptors.clear(); + } + + if (shutdownState == SS_SHUTDOWN_COMMAND) { + StatusData.ShutdownState.CompleteShutdown(); + } + + StatusUpdateCachedDumpIfNecessary(now); +} + TBusSessionImpl::TConnectionsData::TConnectionsData() { } - -void TBusSessionImpl::Act(TConnectionTag) { - TConnectionsGuard guard(ConnectionsLock); - - EShutdownState shutdownState = ConnectionsData.ShutdownState.State.Get(); - if (shutdownState == SS_SHUTDOWN_COMPLETE) { + +void TBusSessionImpl::Act(TConnectionTag) { + TConnectionsGuard guard(ConnectionsLock); + + EShutdownState shutdownState = ConnectionsData.ShutdownState.State.Get(); + if (shutdownState == SS_SHUTDOWN_COMPLETE) { Y_VERIFY(GetRemoveConnectionQueue()->IsEmpty()); Y_VERIFY(GetOnAcceptQueue()->IsEmpty()); - } - - GetRemoveConnectionQueue()->DequeueAllLikelyEmpty(); - GetOnAcceptQueue()->DequeueAllLikelyEmpty(); - - if (shutdownState == SS_SHUTDOWN_COMMAND) { - ConnectionsData.ShutdownState.CompleteShutdown(); - } -} - -void TBusSessionImpl::Listen(int port, TBusMessageQueue* q) { + } + + GetRemoveConnectionQueue()->DequeueAllLikelyEmpty(); + GetOnAcceptQueue()->DequeueAllLikelyEmpty(); + + if (shutdownState == SS_SHUTDOWN_COMMAND) { + ConnectionsData.ShutdownState.CompleteShutdown(); + } +} + +void TBusSessionImpl::Listen(int port, TBusMessageQueue* q) { Listen(BindOnPort(port, Config.ReusePort).second, q); } @@ -494,116 +494,116 @@ void TBusSessionImpl::Listen(const TVector<TBindResult>& bindTo, TBusMessageQueu for (const TBindResult& br : bindTo) { if (actualPort == -1) { actualPort = br.Addr.GetPort(); - } else { + } else { Y_VERIFY(actualPort == br.Addr.GetPort(), "state check"); - } + } if (Config.SocketToS >= 0) { SetSocketToS(*br.Socket, &(br.Addr), Config.SocketToS); } - + TAcceptorPtr acceptor(new TAcceptor(this, ++LastAcceptorId, br.Socket->Release(), br.Addr)); - TConnectionsGuard guard(ConnectionsLock); - InsertAcceptorLockAcquired(acceptor.Get()); + TConnectionsGuard guard(ConnectionsLock); + InsertAcceptorLockAcquired(acceptor.Get()); } - - Config.ListenPort = actualPort; + + Config.ListenPort = actualPort; } -void TBusSessionImpl::SendSnapshotToStatusActor() { +void TBusSessionImpl::SendSnapshotToStatusActor() { //Y_ASSERT(ConnectionsLock.IsLocked()); - + TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> snapshot(new TConnectionsAcceptorsSnapshot); - GetAcceptorsLockAquired(&snapshot->Acceptors); - GetConnectionsLockAquired(&snapshot->Connections); - snapshot->LastAcceptorId = LastAcceptorId; - snapshot->LastConnectionId = LastConnectionId; - StatusData.ConnectionsAcceptorsSnapshotsQueue.Enqueue(snapshot); - GetStatusActor()->Schedule(); -} - -void TBusSessionImpl::InsertConnectionLockAcquired(TRemoteConnection* connection) { + GetAcceptorsLockAquired(&snapshot->Acceptors); + GetConnectionsLockAquired(&snapshot->Connections); + snapshot->LastAcceptorId = LastAcceptorId; + snapshot->LastConnectionId = LastConnectionId; + StatusData.ConnectionsAcceptorsSnapshotsQueue.Enqueue(snapshot); + GetStatusActor()->Schedule(); +} + +void TBusSessionImpl::InsertConnectionLockAcquired(TRemoteConnection* connection) { //Y_ASSERT(ConnectionsLock.IsLocked()); - + Connections.insert(std::make_pair(connection->PeerAddrSocketAddr, connection)); - // connection for given adds may already exist at this point - // (so we overwrite old connection) - // after reconnect, if previous connections wasn't shutdown yet - + // connection for given adds may already exist at this point + // (so we overwrite old connection) + // after reconnect, if previous connections wasn't shutdown yet + bool inserted2 = ConnectionsById.insert(std::make_pair(connection->ConnectionId, connection)).second; Y_VERIFY(inserted2, "state check: must be inserted (2)"); - - SendSnapshotToStatusActor(); -} - -void TBusSessionImpl::InsertAcceptorLockAcquired(TAcceptor* acceptor) { + + SendSnapshotToStatusActor(); +} + +void TBusSessionImpl::InsertAcceptorLockAcquired(TAcceptor* acceptor) { //Y_ASSERT(ConnectionsLock.IsLocked()); - - Acceptors.push_back(acceptor); - - SendSnapshotToStatusActor(); -} - + + Acceptors.push_back(acceptor); + + SendSnapshotToStatusActor(); +} + void TBusSessionImpl::GetConnections(TVector<TRemoteConnectionPtr>* r) { - TConnectionsGuard guard(ConnectionsLock); - GetConnectionsLockAquired(r); -} - + TConnectionsGuard guard(ConnectionsLock); + GetConnectionsLockAquired(r); +} + void TBusSessionImpl::GetAcceptors(TVector<TAcceptorPtr>* r) { - TConnectionsGuard guard(ConnectionsLock); - GetAcceptorsLockAquired(r); -} - + TConnectionsGuard guard(ConnectionsLock); + GetAcceptorsLockAquired(r); +} + void TBusSessionImpl::GetConnectionsLockAquired(TVector<TRemoteConnectionPtr>* r) { //Y_ASSERT(ConnectionsLock.IsLocked()); - - r->reserve(Connections.size()); - + + r->reserve(Connections.size()); + for (auto& connection : Connections) { r->push_back(connection.second); - } -} - + } +} + void TBusSessionImpl::GetAcceptorsLockAquired(TVector<TAcceptorPtr>* r) { //Y_ASSERT(ConnectionsLock.IsLocked()); - - r->reserve(Acceptors.size()); - + + r->reserve(Acceptors.size()); + for (auto& acceptor : Acceptors) { r->push_back(acceptor); - } -} - -TRemoteConnectionPtr TBusSessionImpl::GetConnectionById(ui64 id) { - TConnectionsGuard guard(ConnectionsLock); - + } +} + +TRemoteConnectionPtr TBusSessionImpl::GetConnectionById(ui64 id) { + TConnectionsGuard guard(ConnectionsLock); + THashMap<ui64, TRemoteConnectionPtr>::const_iterator it = ConnectionsById.find(id); - if (it == ConnectionsById.end()) { + if (it == ConnectionsById.end()) { return nullptr; - } else { - return it->second; - } -} - -TAcceptorPtr TBusSessionImpl::GetAcceptorById(ui64 id) { - TGuard<TMutex> guard(ConnectionsLock); - + } else { + return it->second; + } +} + +TAcceptorPtr TBusSessionImpl::GetAcceptorById(ui64 id) { + TGuard<TMutex> guard(ConnectionsLock); + for (const auto& Acceptor : Acceptors) { if (Acceptor->AcceptorId == id) { return Acceptor; - } - } - + } + } + return nullptr; -} - -void TBusSessionImpl::InvokeOnError(TNonDestroyingAutoPtr<TBusMessage> message, EMessageStatus status) { - message->CheckClean(); - ErrorHandler->OnError(message, status); -} - -TRemoteConnectionPtr TBusSessionImpl::GetConnection(const TBusSocketAddr& addr, bool create) { - TConnectionsGuard guard(ConnectionsLock); +} + +void TBusSessionImpl::InvokeOnError(TNonDestroyingAutoPtr<TBusMessage> message, EMessageStatus status) { + message->CheckClean(); + ErrorHandler->OnError(message, status); +} + +TRemoteConnectionPtr TBusSessionImpl::GetConnection(const TBusSocketAddr& addr, bool create) { + TConnectionsGuard guard(ConnectionsLock); TAddrRemoteConnections::const_iterator it = Connections.find(addr); if (it != Connections.end()) { @@ -615,36 +615,36 @@ TRemoteConnectionPtr TBusSessionImpl::GetConnection(const TBusSocketAddr& addr, } Y_VERIFY(IsSource_, "must be source"); - - TRemoteConnectionPtr c(new TRemoteClientConnection(VerifyDynamicCast<TRemoteClientSession*>(this), ++LastConnectionId, addr.ToNetAddr())); - InsertConnectionLockAcquired(c.Get()); + + TRemoteConnectionPtr c(new TRemoteClientConnection(VerifyDynamicCast<TRemoteClientSession*>(this), ++LastConnectionId, addr.ToNetAddr())); + InsertConnectionLockAcquired(c.Get()); return c; } -void TBusSessionImpl::Cron() { +void TBusSessionImpl::Cron() { TVector<TRemoteConnectionPtr> connections; - GetConnections(&connections); - + GetConnections(&connections); + for (const auto& it : connections) { TRemoteConnection* connection = it.Get(); - if (IsSource_) { - VerifyDynamicCast<TRemoteClientConnection*>(connection)->ScheduleTimeoutMessages(); - } else { - VerifyDynamicCast<TRemoteServerConnection*>(connection)->WriterData.TimeToRotateCounters.AddTask(); - // no schedule: do not rotate if there's no traffic - } - } - - // status updates are sent without scheduling - GetStatusActor()->Schedule(); - - Queue->Schedule(IScheduleItemAutoPtr(new TScheduleSession(this, TInstant::Now() + Config.Secret.TimeoutPeriod))); -} - + if (IsSource_) { + VerifyDynamicCast<TRemoteClientConnection*>(connection)->ScheduleTimeoutMessages(); + } else { + VerifyDynamicCast<TRemoteServerConnection*>(connection)->WriterData.TimeToRotateCounters.AddTask(); + // no schedule: do not rotate if there's no traffic + } + } + + // status updates are sent without scheduling + GetStatusActor()->Schedule(); + + Queue->Schedule(IScheduleItemAutoPtr(new TScheduleSession(this, TInstant::Now() + Config.Secret.TimeoutPeriod))); +} + TString TBusSessionImpl::GetNameInternal() { - if (!!Config.Name) { - return Config.Name; - } - return ProtoName; -} + if (!!Config.Name) { + return Config.Name; + } + return ProtoName; +} diff --git a/library/cpp/messagebus/session_impl.h b/library/cpp/messagebus/session_impl.h index 0c6fec1af4..90ef246ff8 100644 --- a/library/cpp/messagebus/session_impl.h +++ b/library/cpp/messagebus/session_impl.h @@ -3,17 +3,17 @@ #include "acceptor_status.h" #include "async_result.h" #include "event_loop.h" -#include "netaddr.h" +#include "netaddr.h" #include "remote_connection.h" -#include "remote_connection_status.h" -#include "session_job_count.h" -#include "shutdown_state.h" +#include "remote_connection_status.h" +#include "session_job_count.h" +#include "shutdown_state.h" #include "ybus.h" #include <library/cpp/messagebus/actor/actor.h> #include <library/cpp/messagebus/actor/queue_in_actor.h> #include <library/cpp/messagebus/monitoring/mon_proto.pb.h> - + #include <library/cpp/threading/future/legacy_future.h> #include <util/generic/array_ref.h> @@ -23,12 +23,12 @@ namespace NBus { namespace NPrivate { typedef TIntrusivePtr<TRemoteClientConnection> TRemoteClientConnectionPtr; typedef TIntrusivePtr<TRemoteServerConnection> TRemoteServerConnectionPtr; - + typedef TIntrusivePtr<TRemoteServerSession> TRemoteServerSessionPtr; typedef TIntrusivePtr<TAcceptor> TAcceptorPtr; typedef TVector<TAcceptorPtr> TAcceptorsPtrs; - + struct TConnectionsAcceptorsSnapshot { TVector<TRemoteConnectionPtr> Connections; TVector<TAcceptorPtr> Acceptors; @@ -37,31 +37,31 @@ namespace NBus { TConnectionsAcceptorsSnapshot(); }; - + typedef TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> TConnectionsAcceptorsSnapshotPtr; - + struct TOnAccept { SOCKET s; TNetAddr addr; TInstant now; }; - + struct TStatusTag {}; struct TConnectionTag {}; - + struct TDeadConnectionTag {}; struct TRemoveTag {}; - + struct TBusSessionImpl : public virtual TBusSession, private ::NActor::TActor<TBusSessionImpl, TStatusTag>, private ::NActor::TActor<TBusSessionImpl, TConnectionTag> - + , private ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionWriterIncrementalStatus, TStatusTag, TDeadConnectionTag>, private ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionReaderIncrementalStatus, TStatusTag, TDeadConnectionTag>, private ::NActor::TQueueInActor<TBusSessionImpl, TAcceptorStatus, TStatusTag, TDeadConnectionTag> - + , private ::NActor::TQueueInActor<TBusSessionImpl, TOnAccept, TConnectionTag>, private ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionPtr, TConnectionTag, TRemoveTag> { @@ -75,43 +75,43 @@ namespace NBus { friend class ::NActor::TQueueInActor<TBusSessionImpl, TAcceptorStatus, TStatusTag, TDeadConnectionTag>; friend class ::NActor::TQueueInActor<TBusSessionImpl, TOnAccept, TConnectionTag>; friend class ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionPtr, TConnectionTag, TRemoveTag>; - + public: ::NActor::TQueueInActor<TBusSessionImpl, TOnAccept, TConnectionTag>* GetOnAcceptQueue() { return this; } - + ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionPtr, TConnectionTag, TRemoveTag>* GetRemoveConnectionQueue() { return this; } - + ::NActor::TActor<TBusSessionImpl, TConnectionTag>* GetConnectionActor() { return this; } - + typedef TGuard<TMutex> TConnectionsGuard; - + TBusSessionImpl(bool isSource, TBusMessageQueue* queue, TBusProtocol* proto, IBusErrorHandler* handler, const TBusSessionConfig& config, const TString& name); - + ~TBusSessionImpl() override; void Shutdown() override; bool IsDown(); - + size_t GetInFlightImpl(const TNetAddr& addr) const; size_t GetConnectSyscallsNumForTestImpl(const TNetAddr& addr) const; void GetInFlightBulk(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const override; void GetConnectSyscallsNumBulkForTest(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const override; - + virtual void FillStatus(); TSessionDumpStatus GetStatusRecordInternal() override; TString GetStatus(ui16 flags = YBUS_STATUS_CONNS) override; TConnectionStatusMonRecord GetStatusProtobuf() override; TString GetStatusSingleLine() override; - + void ProcessItem(TStatusTag, TDeadConnectionTag, const TRemoteConnectionWriterIncrementalStatus&); void ProcessItem(TStatusTag, TDeadConnectionTag, const TRemoteConnectionReaderIncrementalStatus&); void ProcessItem(TStatusTag, TDeadConnectionTag, const TAcceptorStatus&); @@ -128,7 +128,7 @@ namespace NBus { const TBusSessionConfig* GetConfig() const noexcept override; TBusMessageQueue* GetQueue() const noexcept override; TString GetNameInternal() override; - + virtual void OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) = 0; void Listen(int port, TBusMessageQueue* q); @@ -143,106 +143,106 @@ namespace NBus { } typedef THashMap<TBusSocketAddr, TRemoteConnectionPtr> TAddrRemoteConnections; - + void SendSnapshotToStatusActor(); void InsertConnectionLockAcquired(TRemoteConnection* connection); void InsertAcceptorLockAcquired(TAcceptor* acceptor); - + void GetConnections(TVector<TRemoteConnectionPtr>*); void GetAcceptors(TVector<TAcceptorPtr>*); void GetConnectionsLockAquired(TVector<TRemoteConnectionPtr>*); void GetAcceptorsLockAquired(TVector<TAcceptorPtr>*); - + TRemoteConnectionPtr GetConnection(const TBusSocketAddr& addr, bool create); TRemoteConnectionPtr GetConnectionById(ui64 id); TAcceptorPtr GetAcceptorById(ui64 id); - + void InvokeOnError(TNonDestroyingAutoPtr<TBusMessage>, EMessageStatus); void Cron(); - + TBusSessionJobCount JobCount; - + // TODO: replace with actor TMutex ConnectionsLock; - + struct TImpl; THolder<TImpl> Impl; - + const bool IsSource_; - + TBusMessageQueue* const Queue; TBusProtocol* const Proto; // copied to be available after Proto dies const TString ProtoName; - + IBusErrorHandler* const ErrorHandler; TUseCountHolder HandlerUseCountHolder; TBusSessionConfig Config; // TODO: make const - + NEventLoop::TEventLoop WriteEventLoop; NEventLoop::TEventLoop ReadEventLoop; THolder<NThreading::TLegacyFuture<void, false>> ReadEventLoopThread; THolder<NThreading::TLegacyFuture<void, false>> WriteEventLoopThread; - + THashMap<ui64, TRemoteConnectionPtr> ConnectionsById; TAddrRemoteConnections Connections; TAcceptorsPtrs Acceptors; - + struct TStatusData { TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> ConnectionsAcceptorsSnapshot; ::NActor::TQueueForActor<TAtomicSharedPtr<TConnectionsAcceptorsSnapshot>> ConnectionsAcceptorsSnapshotsQueue; TAtomicShutdownState ShutdownState; - + TBusSessionStatus Status; - + TSessionDumpStatus StatusDumpCached; TMutex StatusDumpCachedMutex; TInstant StatusDumpCachedLastUpdate; - + TStatusData(); }; TStatusData StatusData; - + struct TConnectionsData { TAtomicShutdownState ShutdownState; - + TConnectionsData(); }; TConnectionsData ConnectionsData; - + ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionWriterIncrementalStatus, TStatusTag, TDeadConnectionTag>* GetDeadConnectionWriterStatusQueue() { return this; } - + ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionReaderIncrementalStatus, TStatusTag, TDeadConnectionTag>* GetDeadConnectionReaderStatusQueue() { return this; } - + ::NActor::TQueueInActor<TBusSessionImpl, TAcceptorStatus, TStatusTag, TDeadConnectionTag>* GetDeadAcceptorStatusQueue() { return this; } - + template <typename TItem> ::NActor::IQueueInActor<TItem>* GetQueue() { return this; } - + ui64 LastAcceptorId; ui64 LastConnectionId; - + TAtomic Down; TSystemEvent ShutdownCompleteEvent; }; - + inline TBusProtocol* TBusSessionImpl::GetProto() const noexcept { return Proto; } diff --git a/library/cpp/messagebus/session_job_count.cpp b/library/cpp/messagebus/session_job_count.cpp index 768a3f5803..33322b1910 100644 --- a/library/cpp/messagebus/session_job_count.cpp +++ b/library/cpp/messagebus/session_job_count.cpp @@ -1,22 +1,22 @@ #include "session_job_count.h" -#include <util/system/yassert.h> - -using namespace NBus; -using namespace NBus::NPrivate; - -TBusSessionJobCount::TBusSessionJobCount() - : JobCount(0) +#include <util/system/yassert.h> + +using namespace NBus; +using namespace NBus::NPrivate; + +TBusSessionJobCount::TBusSessionJobCount() + : JobCount(0) { } - -TBusSessionJobCount::~TBusSessionJobCount() { + +TBusSessionJobCount::~TBusSessionJobCount() { Y_VERIFY(JobCount == 0, "must be 0 job count to destroy job"); -} - -void TBusSessionJobCount::WaitForZero() { - TGuard<TMutex> guard(Mutex); - while (AtomicGet(JobCount) > 0) { - CondVar.WaitI(Mutex); - } -} +} + +void TBusSessionJobCount::WaitForZero() { + TGuard<TMutex> guard(Mutex); + while (AtomicGet(JobCount) > 0) { + CondVar.WaitI(Mutex); + } +} diff --git a/library/cpp/messagebus/session_job_count.h b/library/cpp/messagebus/session_job_count.h index 4bdec4048f..23aca618b1 100644 --- a/library/cpp/messagebus/session_job_count.h +++ b/library/cpp/messagebus/session_job_count.h @@ -1,39 +1,39 @@ -#pragma once - -#include <util/system/atomic.h> +#pragma once + +#include <util/system/atomic.h> #include <util/system/condvar.h> -#include <util/system/mutex.h> - +#include <util/system/mutex.h> + namespace NBus { namespace NPrivate { class TBusSessionJobCount { private: TAtomic JobCount; - + TMutex Mutex; TCondVar CondVar; - + public: TBusSessionJobCount(); ~TBusSessionJobCount(); - + void Add(unsigned delta) { AtomicAdd(JobCount, delta); } - + void Increment() { Add(1); } - + void Decrement() { if (AtomicDecrement(JobCount) == 0) { TGuard<TMutex> guard(Mutex); CondVar.BroadCast(); } } - + void WaitForZero(); }; - - } + + } } diff --git a/library/cpp/messagebus/shutdown_state.cpp b/library/cpp/messagebus/shutdown_state.cpp index f99b62a6c9..a4e2bfa8b2 100644 --- a/library/cpp/messagebus/shutdown_state.cpp +++ b/library/cpp/messagebus/shutdown_state.cpp @@ -1,20 +1,20 @@ #include "shutdown_state.h" -#include <util/system/yassert.h> - +#include <util/system/yassert.h> + void TAtomicShutdownState::ShutdownCommand() { Y_VERIFY(State.CompareAndSet(SS_RUNNING, SS_SHUTDOWN_COMMAND)); -} - +} + void TAtomicShutdownState::CompleteShutdown() { Y_VERIFY(State.CompareAndSet(SS_SHUTDOWN_COMMAND, SS_SHUTDOWN_COMPLETE)); - ShutdownComplete.Signal(); -} - + ShutdownComplete.Signal(); +} + bool TAtomicShutdownState::IsRunning() { - return State.Get() == SS_RUNNING; -} - + return State.Get() == SS_RUNNING; +} + TAtomicShutdownState::~TAtomicShutdownState() { Y_VERIFY(SS_SHUTDOWN_COMPLETE == State.Get()); -} +} diff --git a/library/cpp/messagebus/shutdown_state.h b/library/cpp/messagebus/shutdown_state.h index 350c87d45d..86bd7110ae 100644 --- a/library/cpp/messagebus/shutdown_state.h +++ b/library/cpp/messagebus/shutdown_state.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include "misc/atomic_box.h" -#include <util/system/event.h> - -enum EShutdownState { - SS_RUNNING, - SS_SHUTDOWN_COMMAND, - SS_SHUTDOWN_COMPLETE, -}; - -struct TAtomicShutdownState { - TAtomicBox<EShutdownState> State; +#include <util/system/event.h> + +enum EShutdownState { + SS_RUNNING, + SS_SHUTDOWN_COMMAND, + SS_SHUTDOWN_COMPLETE, +}; + +struct TAtomicShutdownState { + TAtomicBox<EShutdownState> State; TSystemEvent ShutdownComplete; - - void ShutdownCommand(); - void CompleteShutdown(); - bool IsRunning(); - - ~TAtomicShutdownState(); -}; + + void ShutdownCommand(); + void CompleteShutdown(); + bool IsRunning(); + + ~TAtomicShutdownState(); +}; diff --git a/library/cpp/messagebus/socket_addr.cpp b/library/cpp/messagebus/socket_addr.cpp index ef54da6603..c1b3a28fbe 100644 --- a/library/cpp/messagebus/socket_addr.cpp +++ b/library/cpp/messagebus/socket_addr.cpp @@ -1,23 +1,23 @@ #include "socket_addr.h" - -#include "netaddr.h" - + +#include "netaddr.h" + #include <util/network/address.h> #include <util/network/init.h> #include <util/system/yassert.h> - -using namespace NAddr; - -using namespace NBus; -using namespace NBus::NPrivate; - + +using namespace NAddr; + +using namespace NBus; +using namespace NBus::NPrivate; + static_assert(ADDR_UNSPEC == 0, "expect ADDR_UNSPEC == 0"); - -NBus::NPrivate::TBusSocketAddr::TBusSocketAddr(const NAddr::IRemoteAddr* addr) + +NBus::NPrivate::TBusSocketAddr::TBusSocketAddr(const NAddr::IRemoteAddr* addr) : IPv6ScopeID(0) -{ - const sockaddr* sa = addr->Addr(); - +{ + const sockaddr* sa = addr->Addr(); + switch ((EAddrFamily)sa->sa_family) { case AF_UNSPEC: { IpAddr.Clear(); @@ -37,24 +37,24 @@ NBus::NPrivate::TBusSocketAddr::TBusSocketAddr(const NAddr::IRemoteAddr* addr) } default: Y_FAIL("unknown address family"); - } -} - + } +} + NBus::NPrivate::TBusSocketAddr::TBusSocketAddr(TStringBuf host, unsigned port) { - *this = TNetAddr(host, port); -} - + *this = TNetAddr(host, port); +} + NBus::NPrivate::TBusSocketAddr::TBusSocketAddr(const TNetAddr& addr) { - *this = TBusSocketAddr(&addr); -} - + *this = TBusSocketAddr(&addr); +} + TNetAddr NBus::NPrivate::TBusSocketAddr::ToNetAddr() const { - sockaddr_storage storage; - Zero(storage); - + sockaddr_storage storage; + Zero(storage); + storage.ss_family = (ui16)IpAddr.GetAddrFamily(); - - switch (IpAddr.GetAddrFamily()) { + + switch (IpAddr.GetAddrFamily()) { case ADDR_UNSPEC: return TNetAddr(); case ADDR_IPV4: { @@ -68,12 +68,12 @@ TNetAddr NBus::NPrivate::TBusSocketAddr::ToNetAddr() const { ((sockaddr_in6*)&storage)->sin6_scope_id = HostToInet(IPv6ScopeID); break; } - } - + } + return TNetAddr(new TOpaqueAddr((sockaddr*)&storage)); -} - -template <> +} + +template <> void Out<TBusSocketAddr>(IOutputStream& out, const TBusSocketAddr& addr) { - out << addr.ToNetAddr(); -} + out << addr.ToNetAddr(); +} diff --git a/library/cpp/messagebus/socket_addr.h b/library/cpp/messagebus/socket_addr.h index 7e3df5afbe..959eafe689 100644 --- a/library/cpp/messagebus/socket_addr.h +++ b/library/cpp/messagebus/socket_addr.h @@ -1,18 +1,18 @@ -#pragma once - +#pragma once + #include "hash.h" - + #include <util/generic/hash.h> #include <util/generic/utility.h> -#include <util/network/address.h> +#include <util/network/address.h> #include <util/network/init.h> - + #include <string.h> - -namespace NBus { - class TNetAddr; -} - + +namespace NBus { + class TNetAddr; +} + namespace NBus { namespace NPrivate { enum EAddrFamily { @@ -20,94 +20,94 @@ namespace NBus { ADDR_IPV4 = AF_INET, ADDR_IPV6 = AF_INET6, }; - + class TBusIpAddr { private: EAddrFamily Af; - + union { in_addr In4; in6_addr In6; }; - + public: TBusIpAddr() { Clear(); } - + EAddrFamily GetAddrFamily() const { return Af; } - + void Clear() { Zero(*this); } - + in_addr GetInAddr() const { Y_ASSERT(Af == ADDR_IPV4); return In4; } - + void SetInAddr(const in_addr& in4) { Clear(); Af = ADDR_IPV4; In4 = in4; } - + in6_addr GetIn6Addr() const { Y_ASSERT(Af == ADDR_IPV6); return In6; } - + void SetIn6Addr(const in6_addr& in6) { Clear(); Af = ADDR_IPV6; In6 = in6; } - + bool operator==(const TBusIpAddr& that) const { return memcmp(this, &that, sizeof(that)) == 0; } }; - + class TBusSocketAddr { public: TBusIpAddr IpAddr; ui16 Port; - + //Only makes sense for IPv6 link-local addresses ui32 IPv6ScopeID; - + TBusSocketAddr() : Port(0) , IPv6ScopeID(0) { } - + TBusSocketAddr(const NAddr::IRemoteAddr*); TBusSocketAddr(const TNetAddr&); TBusSocketAddr(TStringBuf host, unsigned port); TNetAddr ToNetAddr() const; - + bool operator==(const TBusSocketAddr& that) const { return IpAddr == that.IpAddr && Port == that.Port; } }; - + } } - -template <> -struct THash<NBus::NPrivate::TBusIpAddr> { - inline size_t operator()(const NBus::NPrivate::TBusIpAddr& a) const { + +template <> +struct THash<NBus::NPrivate::TBusIpAddr> { + inline size_t operator()(const NBus::NPrivate::TBusIpAddr& a) const { return ComputeHash(TStringBuf((const char*)&a, sizeof(a))); - } -}; - -template <> -struct THash<NBus::NPrivate::TBusSocketAddr> { - inline size_t operator()(const NBus::NPrivate::TBusSocketAddr& a) const { - return HashValues(a.IpAddr, a.Port); - } -}; + } +}; + +template <> +struct THash<NBus::NPrivate::TBusSocketAddr> { + inline size_t operator()(const NBus::NPrivate::TBusSocketAddr& a) const { + return HashValues(a.IpAddr, a.Port); + } +}; diff --git a/library/cpp/messagebus/socket_addr_ut.cpp b/library/cpp/messagebus/socket_addr_ut.cpp index a8e4fea47f..783bb62a86 100644 --- a/library/cpp/messagebus/socket_addr_ut.cpp +++ b/library/cpp/messagebus/socket_addr_ut.cpp @@ -1,15 +1,15 @@ #include <library/cpp/testing/unittest/registar.h> - + #include "netaddr.h" -#include "socket_addr.h" - +#include "socket_addr.h" + #include <util/string/cast.h> -using namespace NBus; -using namespace NBus::NPrivate; - +using namespace NBus; +using namespace NBus::NPrivate; + Y_UNIT_TEST_SUITE(TBusSocketAddr) { Y_UNIT_TEST(Simple) { UNIT_ASSERT_VALUES_EQUAL(TString("127.0.0.1:80"), ToString(TBusSocketAddr("127.0.0.1", 80))); - } -} + } +} diff --git a/library/cpp/messagebus/storage.cpp b/library/cpp/messagebus/storage.cpp index f201a4335d..efefc87340 100644 --- a/library/cpp/messagebus/storage.cpp +++ b/library/cpp/messagebus/storage.cpp @@ -1,7 +1,7 @@ #include "storage.h" -#include <typeinfo> - +#include <typeinfo> + namespace NBus { namespace NPrivate { TTimedMessages::TTimedMessages() { @@ -10,13 +10,13 @@ namespace NBus { TTimedMessages::~TTimedMessages() { Y_VERIFY(Items.empty()); } - + void TTimedMessages::PushBack(TNonDestroyingAutoPtr<TBusMessage> m) { TItem i; i.Message.Reset(m.Release()); Items.push_back(i); } - + TNonDestroyingAutoPtr<TBusMessage> TTimedMessages::PopFront() { TBusMessage* r = nullptr; if (!Items.empty()) { @@ -25,11 +25,11 @@ namespace NBus { } return r; } - + bool TTimedMessages::Empty() const { return Items.empty(); } - + size_t TTimedMessages::Size() const { return Items.size(); } @@ -62,7 +62,7 @@ namespace NBus { KeyToMessage.set_empty_key(0); KeyToMessage.set_deleted_key(1); } - + TSyncAckMessages::~TSyncAckMessages() { Y_VERIFY(KeyToMessage.empty()); Y_VERIFY(TimedItems.empty()); @@ -75,14 +75,14 @@ namespace NBus { } TValue value = {m.MessagePtr.Release()}; - + std::pair<TKeyToMessage::iterator, bool> p = KeyToMessage.insert(TKeyToMessage::value_type(m.Header.Id, value)); Y_VERIFY(p.second, "non-unique id; %s", value.Message->Describe().data()); - + TTimedItem item = {m.Header.Id, m.Header.SendTime}; TimedItems.push_back(item); } - + TBusMessage* TSyncAckMessages::Pop(TBusKey id) { TKeyToMessage::iterator it = KeyToMessage.find(id); if (it == KeyToMessage.end()) { @@ -90,9 +90,9 @@ namespace NBus { } TValue v = it->second; KeyToMessage.erase(it); - + // `TimedMessages` still contain record about this message - + return v.Message; } @@ -102,9 +102,9 @@ namespace NBus { Clear(r); return; } - + Y_ASSERT(r->empty()); - + while (!TimedItems.empty()) { TTimedItem i = TimedItems.front(); if (TInstant::MilliSeconds(i.SendTime) > before) { @@ -112,25 +112,25 @@ namespace NBus { } TKeyToMessage::iterator itMessage = KeyToMessage.find(i.Key); - + if (itMessage != KeyToMessage.end()) { r->push_back(itMessage->second.Message); KeyToMessage.erase(itMessage); } - + TimedItems.pop_front(); } - } - + } + void TSyncAckMessages::Clear(TMessagesPtrs* r) { for (TKeyToMessage::const_iterator i = KeyToMessage.begin(); i != KeyToMessage.end(); ++i) { r->push_back(i->second.Message); } - + KeyToMessage.clear(); TimedItems.clear(); - } - + } + void TSyncAckMessages::Gc() { TDeque<TTimedItem> tmp; @@ -140,7 +140,7 @@ namespace NBus { } tmp.push_back(timedItem); } - + TimedItems.swap(tmp); } @@ -151,11 +151,11 @@ namespace NBus { KeyToMessage.erase(it); } } - + void TSyncAckMessages::DumpState() { Cerr << TimedItems.size() << Endl; Cerr << KeyToMessage.size() << Endl; - } - - } + } + + } } diff --git a/library/cpp/messagebus/storage.h b/library/cpp/messagebus/storage.h index f69b2ae857..7d168844ed 100644 --- a/library/cpp/messagebus/storage.h +++ b/library/cpp/messagebus/storage.h @@ -7,13 +7,13 @@ #include <contrib/libs/sparsehash/src/sparsehash/dense_hash_map> #include <util/generic/deque.h> -#include <util/generic/noncopyable.h> -#include <util/generic/utility.h> - +#include <util/generic/noncopyable.h> +#include <util/generic/utility.h> + namespace NBus { namespace NPrivate { typedef TVector<TBusMessage*> TMessagesPtrs; - + class TTimedMessages { public: TTimedMessages(); @@ -21,19 +21,19 @@ namespace NBus { struct TItem { THolder<TBusMessage> Message; - + void Swap(TItem& that) { DoSwap(Message, that.Message); } }; - + typedef TDeque<TMoved<TItem>> TItems; - + void PushBack(TNonDestroyingAutoPtr<TBusMessage> m); TNonDestroyingAutoPtr<TBusMessage> PopFront(); bool Empty() const; size_t Size() const; - + void Timeout(TInstant before, TMessagesPtrs* r); void Clear(TMessagesPtrs* r); @@ -48,9 +48,9 @@ namespace NBus { void Push(TBusMessagePtrAndHeader& m); TBusMessage* Pop(TBusKey id); - + void Timeout(TInstant before, TMessagesPtrs* r); - + void Clear(TMessagesPtrs* r); size_t Size() const { @@ -62,33 +62,33 @@ namespace NBus { void Gc(); void DumpState(); - + private: struct TTimedItem { TBusKey Key; TBusInstant SendTime; }; - + typedef TDeque<TTimedItem> TTimedItems; typedef TDeque<TTimedItem>::iterator TTimedIterator; - + TTimedItems TimedItems; - + struct TValue { TBusMessage* Message; }; - + // keys are already random, no need to hash them further struct TIdHash { size_t operator()(TBusKey value) const { return value; } }; - + typedef google::dense_hash_map<TBusKey, TValue, TIdHash> TKeyToMessage; - + TKeyToMessage KeyToMessage; - }; - + }; + } } diff --git a/library/cpp/messagebus/synchandler.cpp b/library/cpp/messagebus/synchandler.cpp index 4ea4eb1ee0..8e891d66b3 100644 --- a/library/cpp/messagebus/synchandler.cpp +++ b/library/cpp/messagebus/synchandler.cpp @@ -1,10 +1,10 @@ -#include "remote_client_session.h" -#include "remote_connection.h" +#include "remote_client_session.h" +#include "remote_connection.h" #include "ybus.h" -using namespace NBus; -using namespace NBus::NPrivate; - +using namespace NBus; +using namespace NBus::NPrivate; + ///////////////////////////////////////////////////////////////// /// Object that encapsulates all messgae data required for sending /// a message synchronously and receiving a reply. It includes: @@ -27,8 +27,8 @@ struct TBusSyncMessageData { class TSyncHandler: public IBusClientHandler { public: - TSyncHandler(bool expectReply = true) - : ExpectReply(expectReply) + TSyncHandler(bool expectReply = true) + : ExpectReply(expectReply) , Session(nullptr) { } @@ -36,57 +36,57 @@ public: } void OnReply(TAutoPtr<TBusMessage> pMessage0, TAutoPtr<TBusMessage> pReply0) override { - TBusMessage* pMessage = pMessage0.Release(); - TBusMessage* pReply = pReply0.Release(); - + TBusMessage* pMessage = pMessage0.Release(); + TBusMessage* pReply = pReply0.Release(); + if (!ExpectReply) { // Maybe need VERIFY, but it will be better to support backward compatibility here. - return; - } - + return; + } + TBusSyncMessageData* data = static_cast<TBusSyncMessageData*>(pMessage->Data); - SignalResult(data, pReply, MESSAGE_OK); + SignalResult(data, pReply, MESSAGE_OK); } void OnError(TAutoPtr<TBusMessage> pMessage0, EMessageStatus status) override { - TBusMessage* pMessage = pMessage0.Release(); + TBusMessage* pMessage = pMessage0.Release(); TBusSyncMessageData* data = static_cast<TBusSyncMessageData*>(pMessage->Data); if (!data) { return; } SignalResult(data, /*pReply=*/nullptr, status); - } - + } + void OnMessageSent(TBusMessage* pMessage) override { Y_UNUSED(pMessage); Y_ASSERT(ExpectReply); - } - + } + void OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage) override { Y_ASSERT(!ExpectReply); TBusSyncMessageData* data = static_cast<TBusSyncMessageData*>(pMessage.Release()->Data); SignalResult(data, /*pReply=*/nullptr, MESSAGE_OK); - } - - void SetSession(TRemoteClientSession* session) { - if (!ExpectReply) { - Session = session; - } - } - -private: + } + + void SetSession(TRemoteClientSession* session) { + if (!ExpectReply) { + Session = session; + } + } + +private: void SignalResult(TBusSyncMessageData* data, TBusMessage* pReply, EMessageStatus status) const { Y_VERIFY(data, "Message data is set to NULL."); TGuard<TMutex> G(data->ReplyLock); - data->Reply = pReply; + data->Reply = pReply; data->ReplyStatus = status; data->ReplyEvent.Signal(); } - -private: - // This is weird, because in regular client one-way-ness is selected per call, not per session. - bool ExpectReply; - TRemoteClientSession* Session; + +private: + // This is weird, because in regular client one-way-ness is selected per call, not per session. + bool ExpectReply; + TRemoteClientSession* Session; }; namespace NBus { @@ -104,7 +104,7 @@ namespace NBus { public TRemoteClientSession { private: bool NeedReply; - + public: TBusSyncSourceSessionImpl(TBusMessageQueue* queue, TBusProtocol* proto, const TBusClientSessionConfig& config, bool needReply, const TString& name) : TSyncHandler(needReply) @@ -113,16 +113,16 @@ namespace NBus { { SetSession(this); } - + TBusMessage* SendSyncMessage(TBusMessage* pMessage, EMessageStatus& status, const TNetAddr* addr = nullptr) { Y_VERIFY(!Queue->GetExecutor()->IsInExecutorThread(), "SendSyncMessage must not be called from executor thread"); - + TBusMessage* reply = nullptr; THolder<TBusSyncMessageData> data(new TBusSyncMessageData()); - + pMessage->Data = data.Get(); - + { TGuard<TMutex> G(data->ReplyLock); if (NeedReply) { @@ -130,7 +130,7 @@ namespace NBus { } else { status = SendMessageOneWay(pMessage, addr); } - + if (status == MESSAGE_OK) { data->ReplyEvent.Wait(data->ReplyLock); TBusSyncMessageData* rdata = static_cast<TBusSyncMessageData*>(pMessage->Data); @@ -139,7 +139,7 @@ namespace NBus { status = rdata->ReplyStatus; } } - + // deletion of message and reply is a job of application. pMessage->Data = nullptr; @@ -153,46 +153,46 @@ namespace NBus { } } -TBusSyncSourceSession::TBusSyncSourceSession(TIntrusivePtr< ::NBus::NPrivate::TBusSyncSourceSessionImpl> session) - : Session(session) +TBusSyncSourceSession::TBusSyncSourceSession(TIntrusivePtr< ::NBus::NPrivate::TBusSyncSourceSessionImpl> session) + : Session(session) { } - + TBusSyncSourceSession::~TBusSyncSourceSession() { - Shutdown(); -} - -void TBusSyncSourceSession::Shutdown() { - Session->Shutdown(); -} - -TBusMessage* TBusSyncSourceSession::SendSyncMessage(TBusMessage* pMessage, EMessageStatus& status, const TNetAddr* addr) { - return Session->SendSyncMessage(pMessage, status, addr); + Shutdown(); +} + +void TBusSyncSourceSession::Shutdown() { + Session->Shutdown(); +} + +TBusMessage* TBusSyncSourceSession::SendSyncMessage(TBusMessage* pMessage, EMessageStatus& status, const TNetAddr* addr) { + return Session->SendSyncMessage(pMessage, status, addr); } int TBusSyncSourceSession::RegisterService(const char* hostname, TBusKey start, TBusKey end, EIpVersion ipVersion) { - return Session->RegisterService(hostname, start, end, ipVersion); -} + return Session->RegisterService(hostname, start, end, ipVersion); +} -int TBusSyncSourceSession::GetInFlight() { - return Session->GetInFlight(); -} +int TBusSyncSourceSession::GetInFlight() { + return Session->GetInFlight(); +} -const TBusProtocol* TBusSyncSourceSession::GetProto() const { - return Session->GetProto(); -} +const TBusProtocol* TBusSyncSourceSession::GetProto() const { + return Session->GetProto(); +} const TBusClientSession* TBusSyncSourceSession::GetBusClientSessionWorkaroundDoNotUse() const { return Session.Get(); } TBusSyncClientSessionPtr TBusMessageQueue::CreateSyncSource(TBusProtocol* proto, const TBusClientSessionConfig& config, bool needReply, const TString& name) { - TIntrusivePtr<TBusSyncSourceSessionImpl> session = new TBusSyncSourceSessionImpl(this, proto, config, needReply, name); - Add(session.Get()); - return new TBusSyncSourceSession(session); -} + TIntrusivePtr<TBusSyncSourceSessionImpl> session = new TBusSyncSourceSessionImpl(this, proto, config, needReply, name); + Add(session.Get()); + return new TBusSyncSourceSession(session); +} -void TBusMessageQueue::Destroy(TBusSyncClientSessionPtr session) { - Destroy(session->Session.Get()); +void TBusMessageQueue::Destroy(TBusSyncClientSessionPtr session) { + Destroy(session->Session.Get()); Y_UNUSED(session->Session.Release()); } diff --git a/library/cpp/messagebus/test/TestMessageBus.py b/library/cpp/messagebus/test/TestMessageBus.py index 4173c9866e..0bbaa0a313 100644 --- a/library/cpp/messagebus/test/TestMessageBus.py +++ b/library/cpp/messagebus/test/TestMessageBus.py @@ -3,6 +3,6 @@ from devtools.fleur.ytest.integration import UnitTestGroup @group @constraint('library.messagebus') -class TestMessageBus(UnitTestGroup): +class TestMessageBus(UnitTestGroup): def __init__(self, context): UnitTestGroup.__init__(self, context, 'MessageBus', 'library-messagebus-test-ut') diff --git a/library/cpp/messagebus/test/example/client/client.cpp b/library/cpp/messagebus/test/example/client/client.cpp index 3bd9a6f768..89b5f2c9be 100644 --- a/library/cpp/messagebus/test/example/client/client.cpp +++ b/library/cpp/messagebus/test/example/client/client.cpp @@ -1,81 +1,81 @@ #include <library/cpp/messagebus/test/example/common/proto.h> -#include <util/random/random.h> - -using namespace NBus; -using namespace NCalculator; - -namespace NCalculator { +#include <util/random/random.h> + +using namespace NBus; +using namespace NCalculator; + +namespace NCalculator { struct TCalculatorClient: public IBusClientHandler { - TCalculatorProtocol Proto; - TBusMessageQueuePtr MessageQueue; - TBusClientSessionPtr ClientSession; - - TCalculatorClient() { - MessageQueue = CreateMessageQueue(); - TBusClientSessionConfig config; - config.TotalTimeout = 2 * 1000; - ClientSession = TBusClientSession::Create(&Proto, this, config, MessageQueue); - } - + TCalculatorProtocol Proto; + TBusMessageQueuePtr MessageQueue; + TBusClientSessionPtr ClientSession; + + TCalculatorClient() { + MessageQueue = CreateMessageQueue(); + TBusClientSessionConfig config; + config.TotalTimeout = 2 * 1000; + ClientSession = TBusClientSession::Create(&Proto, this, config, MessageQueue); + } + ~TCalculatorClient() override { - MessageQueue->Stop(); - } - + MessageQueue->Stop(); + } + void OnReply(TAutoPtr<TBusMessage> request, TAutoPtr<TBusMessage> response0) override { Y_VERIFY(response0->GetHeader()->Type == TResponse::MessageType, "wrong response"); - TResponse* response = VerifyDynamicCast<TResponse*>(response0.Get()); - if (request->GetHeader()->Type == TRequestSum::MessageType) { - TRequestSum* requestSum = VerifyDynamicCast<TRequestSum*>(request.Get()); - int a = requestSum->Record.GetA(); - int b = requestSum->Record.GetB(); - Cerr << a << " + " << b << " = " << response->Record.GetResult() << "\n"; - } else if (request->GetHeader()->Type == TRequestMul::MessageType) { - TRequestMul* requestMul = VerifyDynamicCast<TRequestMul*>(request.Get()); - int a = requestMul->Record.GetA(); - int b = requestMul->Record.GetB(); - Cerr << a << " * " << b << " = " << response->Record.GetResult() << "\n"; - } else { + TResponse* response = VerifyDynamicCast<TResponse*>(response0.Get()); + if (request->GetHeader()->Type == TRequestSum::MessageType) { + TRequestSum* requestSum = VerifyDynamicCast<TRequestSum*>(request.Get()); + int a = requestSum->Record.GetA(); + int b = requestSum->Record.GetB(); + Cerr << a << " + " << b << " = " << response->Record.GetResult() << "\n"; + } else if (request->GetHeader()->Type == TRequestMul::MessageType) { + TRequestMul* requestMul = VerifyDynamicCast<TRequestMul*>(request.Get()); + int a = requestMul->Record.GetA(); + int b = requestMul->Record.GetB(); + Cerr << a << " * " << b << " = " << response->Record.GetResult() << "\n"; + } else { Y_FAIL("unknown request"); - } - } - + } + } + void OnError(TAutoPtr<TBusMessage>, EMessageStatus status) override { - Cerr << "got error " << status << "\n"; - } - }; - -} - -int main(int, char**) { - TCalculatorClient client; - - for (;;) { - TNetAddr addr(TNetAddr("127.0.0.1", TCalculatorProtocol().GetPort())); - - int a = RandomNumber<unsigned>(10); - int b = RandomNumber<unsigned>(10); - EMessageStatus ok; - if (RandomNumber<bool>()) { - TAutoPtr<TRequestSum> request(new TRequestSum); - request->Record.SetA(a); - request->Record.SetB(b); - Cerr << "sending " << a << " + " << b << "\n"; - ok = client.ClientSession->SendMessageAutoPtr(request, &addr); - } else { - TAutoPtr<TRequestMul> request(new TRequestMul); - request->Record.SetA(a); - request->Record.SetB(b); - Cerr << "sending " << a << " * " << b << "\n"; - ok = client.ClientSession->SendMessageAutoPtr(request, &addr); - } - - if (ok != MESSAGE_OK) { - Cerr << "failed to send message " << ok << "\n"; - } - - Sleep(TDuration::Seconds(1)); - } - - return 0; -} + Cerr << "got error " << status << "\n"; + } + }; + +} + +int main(int, char**) { + TCalculatorClient client; + + for (;;) { + TNetAddr addr(TNetAddr("127.0.0.1", TCalculatorProtocol().GetPort())); + + int a = RandomNumber<unsigned>(10); + int b = RandomNumber<unsigned>(10); + EMessageStatus ok; + if (RandomNumber<bool>()) { + TAutoPtr<TRequestSum> request(new TRequestSum); + request->Record.SetA(a); + request->Record.SetB(b); + Cerr << "sending " << a << " + " << b << "\n"; + ok = client.ClientSession->SendMessageAutoPtr(request, &addr); + } else { + TAutoPtr<TRequestMul> request(new TRequestMul); + request->Record.SetA(a); + request->Record.SetB(b); + Cerr << "sending " << a << " * " << b << "\n"; + ok = client.ClientSession->SendMessageAutoPtr(request, &addr); + } + + if (ok != MESSAGE_OK) { + Cerr << "failed to send message " << ok << "\n"; + } + + Sleep(TDuration::Seconds(1)); + } + + return 0; +} diff --git a/library/cpp/messagebus/test/example/client/ya.make b/library/cpp/messagebus/test/example/client/ya.make index 81713a5318..a660a01698 100644 --- a/library/cpp/messagebus/test/example/client/ya.make +++ b/library/cpp/messagebus/test/example/client/ya.make @@ -1,13 +1,13 @@ -PROGRAM(messagebus_example_client) - +PROGRAM(messagebus_example_client) + OWNER(g:messagebus) - -PEERDIR( + +PEERDIR( library/cpp/messagebus/test/example/common -) - -SRCS( - client.cpp -) - -END() +) + +SRCS( + client.cpp +) + +END() diff --git a/library/cpp/messagebus/test/example/common/messages.proto b/library/cpp/messagebus/test/example/common/messages.proto index 12cdf38fb5..16b858fc77 100644 --- a/library/cpp/messagebus/test/example/common/messages.proto +++ b/library/cpp/messagebus/test/example/common/messages.proto @@ -1,15 +1,15 @@ -package NCalculator; - -message TRequestSumRecord { - required int32 A = 1; - required int32 B = 2; -} - -message TRequestMulRecord { - required int32 A = 1; - required int32 B = 2; -} - -message TResponseRecord { - required int32 Result = 1; -} +package NCalculator; + +message TRequestSumRecord { + required int32 A = 1; + required int32 B = 2; +} + +message TRequestMulRecord { + required int32 A = 1; + required int32 B = 2; +} + +message TResponseRecord { + required int32 Result = 1; +} diff --git a/library/cpp/messagebus/test/example/common/proto.cpp b/library/cpp/messagebus/test/example/common/proto.cpp index 3531e3d06c..1d18aa77ea 100644 --- a/library/cpp/messagebus/test/example/common/proto.cpp +++ b/library/cpp/messagebus/test/example/common/proto.cpp @@ -1,12 +1,12 @@ -#include "proto.h" - -using namespace NCalculator; -using namespace NBus; - -TCalculatorProtocol::TCalculatorProtocol() - : TBusBufferProtocol("Calculator", 34567) -{ - RegisterType(new TRequestSum); - RegisterType(new TRequestMul); - RegisterType(new TResponse); -} +#include "proto.h" + +using namespace NCalculator; +using namespace NBus; + +TCalculatorProtocol::TCalculatorProtocol() + : TBusBufferProtocol("Calculator", 34567) +{ + RegisterType(new TRequestSum); + RegisterType(new TRequestMul); + RegisterType(new TResponse); +} diff --git a/library/cpp/messagebus/test/example/common/proto.h b/library/cpp/messagebus/test/example/common/proto.h index f9fbd5ce56..a151aac468 100644 --- a/library/cpp/messagebus/test/example/common/proto.h +++ b/library/cpp/messagebus/test/example/common/proto.h @@ -1,17 +1,17 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/test/example/common/messages.pb.h> - + #include <library/cpp/messagebus/ybus.h> #include <library/cpp/messagebus/protobuf/ybusbuf.h> -namespace NCalculator { - typedef ::NBus::TBusBufferMessage<TRequestSumRecord, 1> TRequestSum; - typedef ::NBus::TBusBufferMessage<TRequestMulRecord, 2> TRequestMul; - typedef ::NBus::TBusBufferMessage<TResponseRecord, 3> TResponse; - +namespace NCalculator { + typedef ::NBus::TBusBufferMessage<TRequestSumRecord, 1> TRequestSum; + typedef ::NBus::TBusBufferMessage<TRequestMulRecord, 2> TRequestMul; + typedef ::NBus::TBusBufferMessage<TResponseRecord, 3> TResponse; + struct TCalculatorProtocol: public ::NBus::TBusBufferProtocol { - TCalculatorProtocol(); - }; - -} + TCalculatorProtocol(); + }; + +} diff --git a/library/cpp/messagebus/test/example/common/ya.make b/library/cpp/messagebus/test/example/common/ya.make index 14f48ff6c0..4da16608fc 100644 --- a/library/cpp/messagebus/test/example/common/ya.make +++ b/library/cpp/messagebus/test/example/common/ya.make @@ -1,15 +1,15 @@ -LIBRARY(messagebus_test_example_common) - +LIBRARY(messagebus_test_example_common) + OWNER(g:messagebus) - -PEERDIR( + +PEERDIR( library/cpp/messagebus library/cpp/messagebus/protobuf -) - -SRCS( - proto.cpp - messages.proto -) - -END() +) + +SRCS( + proto.cpp + messages.proto +) + +END() diff --git a/library/cpp/messagebus/test/example/server/server.cpp b/library/cpp/messagebus/test/example/server/server.cpp index a080f3548b..13e52d75f5 100644 --- a/library/cpp/messagebus/test/example/server/server.cpp +++ b/library/cpp/messagebus/test/example/server/server.cpp @@ -1,58 +1,58 @@ #include <library/cpp/messagebus/test/example/common/proto.h> - -using namespace NBus; -using namespace NCalculator; - -namespace NCalculator { + +using namespace NBus; +using namespace NCalculator; + +namespace NCalculator { struct TCalculatorServer: public IBusServerHandler { - TCalculatorProtocol Proto; - TBusMessageQueuePtr MessageQueue; - TBusServerSessionPtr ServerSession; - - TCalculatorServer() { - MessageQueue = CreateMessageQueue(); - TBusServerSessionConfig config; - ServerSession = TBusServerSession::Create(&Proto, this, config, MessageQueue); - } - + TCalculatorProtocol Proto; + TBusMessageQueuePtr MessageQueue; + TBusServerSessionPtr ServerSession; + + TCalculatorServer() { + MessageQueue = CreateMessageQueue(); + TBusServerSessionConfig config; + ServerSession = TBusServerSession::Create(&Proto, this, config, MessageQueue); + } + ~TCalculatorServer() override { - MessageQueue->Stop(); - } - + MessageQueue->Stop(); + } + void OnMessage(TOnMessageContext& request) override { - if (request.GetMessage()->GetHeader()->Type == TRequestSum::MessageType) { - TRequestSum* requestSum = VerifyDynamicCast<TRequestSum*>(request.GetMessage()); - int a = requestSum->Record.GetA(); - int b = requestSum->Record.GetB(); - int result = a + b; - Cerr << "requested " << a << " + " << b << ", sending " << result << "\n"; - TAutoPtr<TResponse> response(new TResponse); - response->Record.SetResult(result); - request.SendReplyMove(response); - } else if (request.GetMessage()->GetHeader()->Type == TRequestMul::MessageType) { - TRequestMul* requestMul = VerifyDynamicCast<TRequestMul*>(request.GetMessage()); - int a = requestMul->Record.GetA(); - int b = requestMul->Record.GetB(); - int result = a * b; - Cerr << "requested " << a << " * " << b << ", sending " << result << "\n"; - TAutoPtr<TResponse> response(new TResponse); - response->Record.SetResult(result); - request.SendReplyMove(response); - } else { + if (request.GetMessage()->GetHeader()->Type == TRequestSum::MessageType) { + TRequestSum* requestSum = VerifyDynamicCast<TRequestSum*>(request.GetMessage()); + int a = requestSum->Record.GetA(); + int b = requestSum->Record.GetB(); + int result = a + b; + Cerr << "requested " << a << " + " << b << ", sending " << result << "\n"; + TAutoPtr<TResponse> response(new TResponse); + response->Record.SetResult(result); + request.SendReplyMove(response); + } else if (request.GetMessage()->GetHeader()->Type == TRequestMul::MessageType) { + TRequestMul* requestMul = VerifyDynamicCast<TRequestMul*>(request.GetMessage()); + int a = requestMul->Record.GetA(); + int b = requestMul->Record.GetB(); + int result = a * b; + Cerr << "requested " << a << " * " << b << ", sending " << result << "\n"; + TAutoPtr<TResponse> response(new TResponse); + response->Record.SetResult(result); + request.SendReplyMove(response); + } else { Y_FAIL("unknown request"); - } - } - }; + } + } + }; +} + +int main(int, char**) { + TCalculatorServer server; + + Cerr << "listening on port " << server.ServerSession->GetActualListenPort() << "\n"; + + for (;;) { + Sleep(TDuration::Seconds(1)); + } + + return 0; } - -int main(int, char**) { - TCalculatorServer server; - - Cerr << "listening on port " << server.ServerSession->GetActualListenPort() << "\n"; - - for (;;) { - Sleep(TDuration::Seconds(1)); - } - - return 0; -} diff --git a/library/cpp/messagebus/test/example/server/ya.make b/library/cpp/messagebus/test/example/server/ya.make index 3bf4c31853..8cdd97cb12 100644 --- a/library/cpp/messagebus/test/example/server/ya.make +++ b/library/cpp/messagebus/test/example/server/ya.make @@ -1,13 +1,13 @@ -PROGRAM(messagebus_example_server) - +PROGRAM(messagebus_example_server) + OWNER(g:messagebus) - -PEERDIR( + +PEERDIR( library/cpp/messagebus/test/example/common -) - -SRCS( - server.cpp -) - -END() +) + +SRCS( + server.cpp +) + +END() diff --git a/library/cpp/messagebus/test/example/ya.make b/library/cpp/messagebus/test/example/ya.make index 972458d255..f275351c29 100644 --- a/library/cpp/messagebus/test/example/ya.make +++ b/library/cpp/messagebus/test/example/ya.make @@ -1,7 +1,7 @@ OWNER(g:messagebus) - -RECURSE( + +RECURSE( client common server -) +) diff --git a/library/cpp/messagebus/test/helper/alloc_counter.h b/library/cpp/messagebus/test/helper/alloc_counter.h index 88db651e69..ec9041cb15 100644 --- a/library/cpp/messagebus/test/helper/alloc_counter.h +++ b/library/cpp/messagebus/test/helper/alloc_counter.h @@ -1,21 +1,21 @@ -#pragma once - -#include <util/generic/noncopyable.h> -#include <util/system/atomic.h> -#include <util/system/yassert.h> - +#pragma once + +#include <util/generic/noncopyable.h> +#include <util/system/atomic.h> +#include <util/system/yassert.h> + class TAllocCounter : TNonCopyable { -private: - TAtomic* CountPtr; +private: + TAtomic* CountPtr; -public: +public: TAllocCounter(TAtomic* countPtr) : CountPtr(countPtr) { - AtomicIncrement(*CountPtr); - } - - ~TAllocCounter() { + AtomicIncrement(*CountPtr); + } + + ~TAllocCounter() { Y_VERIFY(AtomicDecrement(*CountPtr) >= 0, "released too many"); - } -}; + } +}; diff --git a/library/cpp/messagebus/test/helper/example.cpp b/library/cpp/messagebus/test/helper/example.cpp index a1913b58c1..7c6d704042 100644 --- a/library/cpp/messagebus/test/helper/example.cpp +++ b/library/cpp/messagebus/test/helper/example.cpp @@ -1,281 +1,281 @@ #include <library/cpp/testing/unittest/registar.h> - + #include "example.h" -#include <util/generic/cast.h> - -using namespace NBus; -using namespace NBus::NTest; - +#include <util/generic/cast.h> + +using namespace NBus; +using namespace NBus::NTest; + static void FillWithJunk(TArrayRef<char> data) { TStringBuf junk = "01234567890123456789012345678901234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789012345678901234567890123456789"; - + for (size_t i = 0; i < data.size(); i += junk.size()) { memcpy(data.data() + i, junk.data(), Min(junk.size(), data.size() - i)); - } -} - + } +} + static TString JunkString(size_t len) { - TTempBuf temp(len); + TTempBuf temp(len); TArrayRef<char> tempArrayRef(temp.Data(), len); - FillWithJunk(tempArrayRef); - + FillWithJunk(tempArrayRef); + return TString(tempArrayRef.data(), tempArrayRef.size()); -} - -TExampleRequest::TExampleRequest(TAtomic* counterPtr, size_t payloadSize) - : TBusMessage(77) - , AllocCounter(counterPtr) - , Data(JunkString(payloadSize)) -{ -} - -TExampleRequest::TExampleRequest(ECreateUninitialized, TAtomic* counterPtr) - : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) - , AllocCounter(counterPtr) +} + +TExampleRequest::TExampleRequest(TAtomic* counterPtr, size_t payloadSize) + : TBusMessage(77) + , AllocCounter(counterPtr) + , Data(JunkString(payloadSize)) { } - -TExampleResponse::TExampleResponse(TAtomic* counterPtr, size_t payloadSize) - : TBusMessage(79) - , AllocCounter(counterPtr) - , Data(JunkString(payloadSize)) -{ -} - -TExampleResponse::TExampleResponse(ECreateUninitialized, TAtomic* counterPtr) - : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) - , AllocCounter(counterPtr) + +TExampleRequest::TExampleRequest(ECreateUninitialized, TAtomic* counterPtr) + : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) + , AllocCounter(counterPtr) { } - -TExampleProtocol::TExampleProtocol(int port) - : TBusProtocol("Example", port) - , RequestCount(0) - , ResponseCount(0) - , RequestCountDeserialized(0) - , ResponseCountDeserialized(0) - , StartCount(0) + +TExampleResponse::TExampleResponse(TAtomic* counterPtr, size_t payloadSize) + : TBusMessage(79) + , AllocCounter(counterPtr) + , Data(JunkString(payloadSize)) { } - -TExampleProtocol::~TExampleProtocol() { - if (UncaughtException()) { - // so it could be reported in test - return; - } + +TExampleResponse::TExampleResponse(ECreateUninitialized, TAtomic* counterPtr) + : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) + , AllocCounter(counterPtr) +{ +} + +TExampleProtocol::TExampleProtocol(int port) + : TBusProtocol("Example", port) + , RequestCount(0) + , ResponseCount(0) + , RequestCountDeserialized(0) + , ResponseCountDeserialized(0) + , StartCount(0) +{ +} + +TExampleProtocol::~TExampleProtocol() { + if (UncaughtException()) { + // so it could be reported in test + return; + } Y_VERIFY(0 == AtomicGet(RequestCount), "protocol %s: must be 0 requests allocated, actually %d", GetService(), int(RequestCount)); Y_VERIFY(0 == AtomicGet(ResponseCount), "protocol %s: must be 0 responses allocated, actually %d", GetService(), int(ResponseCount)); Y_VERIFY(0 == AtomicGet(RequestCountDeserialized), "protocol %s: must be 0 requests deserialized allocated, actually %d", GetService(), int(RequestCountDeserialized)); Y_VERIFY(0 == AtomicGet(ResponseCountDeserialized), "protocol %s: must be 0 responses deserialized allocated, actually %d", GetService(), int(ResponseCountDeserialized)); Y_VERIFY(0 == AtomicGet(StartCount), "protocol %s: must be 0 start objects allocated, actually %d", GetService(), int(StartCount)); -} - -void TExampleProtocol::Serialize(const TBusMessage* message, TBuffer& buffer) { - // Messages have no data, we recreate them from scratch - // instead of sending, so we don't need to serialize them. - if (const TExampleRequest* exampleMessage = dynamic_cast<const TExampleRequest*>(message)) { +} + +void TExampleProtocol::Serialize(const TBusMessage* message, TBuffer& buffer) { + // Messages have no data, we recreate them from scratch + // instead of sending, so we don't need to serialize them. + if (const TExampleRequest* exampleMessage = dynamic_cast<const TExampleRequest*>(message)) { buffer.Append(exampleMessage->Data.data(), exampleMessage->Data.size()); - } else if (const TExampleResponse* exampleReply = dynamic_cast<const TExampleResponse*>(message)) { + } else if (const TExampleResponse* exampleReply = dynamic_cast<const TExampleResponse*>(message)) { buffer.Append(exampleReply->Data.data(), exampleReply->Data.size()); - } else { + } else { Y_FAIL("unknown message type"); - } -} - + } +} + TAutoPtr<TBusMessage> TExampleProtocol::Deserialize(ui16 messageType, TArrayRef<const char> payload) { - // TODO: check data + // TODO: check data Y_UNUSED(payload); - - if (messageType == 77) { - TExampleRequest* exampleMessage = new TExampleRequest(MESSAGE_CREATE_UNINITIALIZED, &RequestCountDeserialized); - exampleMessage->Data.append(payload.data(), payload.size()); - return exampleMessage; - } else if (messageType == 79) { - TExampleResponse* exampleReply = new TExampleResponse(MESSAGE_CREATE_UNINITIALIZED, &ResponseCountDeserialized); - exampleReply->Data.append(payload.data(), payload.size()); - return exampleReply; - } else { + + if (messageType == 77) { + TExampleRequest* exampleMessage = new TExampleRequest(MESSAGE_CREATE_UNINITIALIZED, &RequestCountDeserialized); + exampleMessage->Data.append(payload.data(), payload.size()); + return exampleMessage; + } else if (messageType == 79) { + TExampleResponse* exampleReply = new TExampleResponse(MESSAGE_CREATE_UNINITIALIZED, &ResponseCountDeserialized); + exampleReply->Data.append(payload.data(), payload.size()); + return exampleReply; + } else { return nullptr; - } -} - -TExampleClient::TExampleClient(const TBusClientSessionConfig sessionConfig, int port) - : Proto(port) - , UseCompression(false) - , CrashOnError(false) - , DataSize(320) - , MessageCount(0) - , RepliesCount(0) - , Errors(0) - , LastError(MESSAGE_OK) -{ - Bus = CreateMessageQueue("TExampleClient"); - - Session = TBusClientSession::Create(&Proto, this, sessionConfig, Bus); - - Session->RegisterService("localhost"); -} - -TExampleClient::~TExampleClient() { -} - + } +} + +TExampleClient::TExampleClient(const TBusClientSessionConfig sessionConfig, int port) + : Proto(port) + , UseCompression(false) + , CrashOnError(false) + , DataSize(320) + , MessageCount(0) + , RepliesCount(0) + , Errors(0) + , LastError(MESSAGE_OK) +{ + Bus = CreateMessageQueue("TExampleClient"); + + Session = TBusClientSession::Create(&Proto, this, sessionConfig, Bus); + + Session->RegisterService("localhost"); +} + +TExampleClient::~TExampleClient() { +} + EMessageStatus TExampleClient::SendMessage(const TNetAddr* addr) { - TAutoPtr<TExampleRequest> message(new TExampleRequest(&Proto.RequestCount, DataSize)); - message->SetCompressed(UseCompression); - return Session->SendMessageAutoPtr(message, addr); -} - + TAutoPtr<TExampleRequest> message(new TExampleRequest(&Proto.RequestCount, DataSize)); + message->SetCompressed(UseCompression); + return Session->SendMessageAutoPtr(message, addr); +} + void TExampleClient::SendMessages(size_t count, const TNetAddr* addr) { - UNIT_ASSERT(MessageCount == 0); - UNIT_ASSERT(RepliesCount == 0); - UNIT_ASSERT(Errors == 0); - - WorkDone.Reset(); - MessageCount = count; - for (ssize_t i = 0; i < MessageCount; ++i) { - EMessageStatus s = SendMessage(addr); - UNIT_ASSERT_EQUAL_C(s, MESSAGE_OK, "expecting OK, got " << s); - } -} - -void TExampleClient::SendMessages(size_t count, const TNetAddr& addr) { - SendMessages(count, &addr); -} - -void TExampleClient::ResetCounters() { - MessageCount = 0; - RepliesCount = 0; - Errors = 0; - LastError = MESSAGE_OK; - - WorkDone.Reset(); -} - -void TExampleClient::WaitReplies() { - WorkDone.WaitT(TDuration::Seconds(60)); - - UNIT_ASSERT_VALUES_EQUAL(AtomicGet(RepliesCount), MessageCount); - UNIT_ASSERT_VALUES_EQUAL(AtomicGet(Errors), 0); - UNIT_ASSERT_VALUES_EQUAL(Session->GetInFlight(), 0); - - ResetCounters(); -} - + UNIT_ASSERT(MessageCount == 0); + UNIT_ASSERT(RepliesCount == 0); + UNIT_ASSERT(Errors == 0); + + WorkDone.Reset(); + MessageCount = count; + for (ssize_t i = 0; i < MessageCount; ++i) { + EMessageStatus s = SendMessage(addr); + UNIT_ASSERT_EQUAL_C(s, MESSAGE_OK, "expecting OK, got " << s); + } +} + +void TExampleClient::SendMessages(size_t count, const TNetAddr& addr) { + SendMessages(count, &addr); +} + +void TExampleClient::ResetCounters() { + MessageCount = 0; + RepliesCount = 0; + Errors = 0; + LastError = MESSAGE_OK; + + WorkDone.Reset(); +} + +void TExampleClient::WaitReplies() { + WorkDone.WaitT(TDuration::Seconds(60)); + + UNIT_ASSERT_VALUES_EQUAL(AtomicGet(RepliesCount), MessageCount); + UNIT_ASSERT_VALUES_EQUAL(AtomicGet(Errors), 0); + UNIT_ASSERT_VALUES_EQUAL(Session->GetInFlight(), 0); + + ResetCounters(); +} + EMessageStatus TExampleClient::WaitForError() { - WorkDone.WaitT(TDuration::Seconds(60)); - - UNIT_ASSERT_VALUES_EQUAL(1, MessageCount); - UNIT_ASSERT_VALUES_EQUAL(0, AtomicGet(RepliesCount)); - UNIT_ASSERT_VALUES_EQUAL(0, Session->GetInFlight()); - UNIT_ASSERT_VALUES_EQUAL(1, Errors); + WorkDone.WaitT(TDuration::Seconds(60)); + + UNIT_ASSERT_VALUES_EQUAL(1, MessageCount); + UNIT_ASSERT_VALUES_EQUAL(0, AtomicGet(RepliesCount)); + UNIT_ASSERT_VALUES_EQUAL(0, Session->GetInFlight()); + UNIT_ASSERT_VALUES_EQUAL(1, Errors); EMessageStatus result = LastError; - - ResetCounters(); + + ResetCounters(); return result; -} - +} + void TExampleClient::WaitForError(EMessageStatus status) { EMessageStatus error = WaitForError(); UNIT_ASSERT_VALUES_EQUAL(status, error); } void TExampleClient::SendMessagesWaitReplies(size_t count, const TNetAddr* addr) { - SendMessages(count, addr); - WaitReplies(); -} - -void TExampleClient::SendMessagesWaitReplies(size_t count, const TNetAddr& addr) { - SendMessagesWaitReplies(count, &addr); -} - -void TExampleClient::OnReply(TAutoPtr<TBusMessage> mess, TAutoPtr<TBusMessage> reply) { + SendMessages(count, addr); + WaitReplies(); +} + +void TExampleClient::SendMessagesWaitReplies(size_t count, const TNetAddr& addr) { + SendMessagesWaitReplies(count, &addr); +} + +void TExampleClient::OnReply(TAutoPtr<TBusMessage> mess, TAutoPtr<TBusMessage> reply) { Y_UNUSED(mess); Y_UNUSED(reply); - - if (AtomicIncrement(RepliesCount) == MessageCount) { - WorkDone.Signal(); - } -} - -void TExampleClient::OnError(TAutoPtr<TBusMessage> mess, EMessageStatus status) { - if (CrashOnError) { + + if (AtomicIncrement(RepliesCount) == MessageCount) { + WorkDone.Signal(); + } +} + +void TExampleClient::OnError(TAutoPtr<TBusMessage> mess, EMessageStatus status) { + if (CrashOnError) { Y_FAIL("client failed: %s", ToCString(status)); - } - + } + Y_UNUSED(mess); - - AtomicIncrement(Errors); - LastError = status; - WorkDone.Signal(); -} - -TExampleServer::TExampleServer( + + AtomicIncrement(Errors); + LastError = status; + WorkDone.Signal(); +} + +TExampleServer::TExampleServer( const char* name, const TBusServerSessionConfig& sessionConfig) - : UseCompression(false) - , AckMessageBeforeSendReply(false) - , ForgetRequest(false) -{ - Bus = CreateMessageQueue(name); - Session = TBusServerSession::Create(&Proto, this, sessionConfig, Bus); -} - -TExampleServer::TExampleServer(unsigned port, const char* name) - : UseCompression(false) - , AckMessageBeforeSendReply(false) - , ForgetRequest(false) -{ - Bus = CreateMessageQueue(name); - TBusServerSessionConfig sessionConfig; - sessionConfig.ListenPort = port; - Session = TBusServerSession::Create(&Proto, this, sessionConfig, Bus); -} - -TExampleServer::~TExampleServer() { -} - -size_t TExampleServer::GetInFlight() const { - return Session->GetInFlight(); -} - -unsigned TExampleServer::GetActualListenPort() const { - return Session->GetActualListenPort(); -} - -TNetAddr TExampleServer::GetActualListenAddr() const { - return TNetAddr("127.0.0.1", GetActualListenPort()); -} - -void TExampleServer::WaitForOnMessageCount(unsigned n) { - TestSync.WaitFor(n); -} - -void TExampleServer::OnMessage(TOnMessageContext& mess) { - TestSync.Inc(); - - TExampleRequest* request = VerifyDynamicCast<TExampleRequest*>(mess.GetMessage()); - - if (ForgetRequest) { - mess.ForgetRequest(); - return; - } - - TAutoPtr<TBusMessage> reply(new TExampleResponse(&Proto.ResponseCount, DataSize.GetOrElse(request->Data.size()))); - reply->SetCompressed(UseCompression); - - EMessageStatus status; - if (AckMessageBeforeSendReply) { - TBusIdentity ident; - mess.AckMessage(ident); - status = Session->SendReply(ident, reply.Release()); // TODO: leaks on error - } else { - status = mess.SendReplyMove(reply); - } - + : UseCompression(false) + , AckMessageBeforeSendReply(false) + , ForgetRequest(false) +{ + Bus = CreateMessageQueue(name); + Session = TBusServerSession::Create(&Proto, this, sessionConfig, Bus); +} + +TExampleServer::TExampleServer(unsigned port, const char* name) + : UseCompression(false) + , AckMessageBeforeSendReply(false) + , ForgetRequest(false) +{ + Bus = CreateMessageQueue(name); + TBusServerSessionConfig sessionConfig; + sessionConfig.ListenPort = port; + Session = TBusServerSession::Create(&Proto, this, sessionConfig, Bus); +} + +TExampleServer::~TExampleServer() { +} + +size_t TExampleServer::GetInFlight() const { + return Session->GetInFlight(); +} + +unsigned TExampleServer::GetActualListenPort() const { + return Session->GetActualListenPort(); +} + +TNetAddr TExampleServer::GetActualListenAddr() const { + return TNetAddr("127.0.0.1", GetActualListenPort()); +} + +void TExampleServer::WaitForOnMessageCount(unsigned n) { + TestSync.WaitFor(n); +} + +void TExampleServer::OnMessage(TOnMessageContext& mess) { + TestSync.Inc(); + + TExampleRequest* request = VerifyDynamicCast<TExampleRequest*>(mess.GetMessage()); + + if (ForgetRequest) { + mess.ForgetRequest(); + return; + } + + TAutoPtr<TBusMessage> reply(new TExampleResponse(&Proto.ResponseCount, DataSize.GetOrElse(request->Data.size()))); + reply->SetCompressed(UseCompression); + + EMessageStatus status; + if (AckMessageBeforeSendReply) { + TBusIdentity ident; + mess.AckMessage(ident); + status = Session->SendReply(ident, reply.Release()); // TODO: leaks on error + } else { + status = mess.SendReplyMove(reply); + } + Y_VERIFY(status == MESSAGE_OK, "failed to send reply: %s", ToString(status).data()); -} +} diff --git a/library/cpp/messagebus/test/helper/example.h b/library/cpp/messagebus/test/helper/example.h index 1ff7d16c1a..26b7475308 100644 --- a/library/cpp/messagebus/test/helper/example.h +++ b/library/cpp/messagebus/test/helper/example.h @@ -1,9 +1,9 @@ #pragma once #include <library/cpp/testing/unittest/registar.h> - -#include "alloc_counter.h" -#include "message_handler_error.h" + +#include "alloc_counter.h" +#include "message_handler_error.h" #include <library/cpp/messagebus/ybus.h> #include <library/cpp/messagebus/misc/test_sync.h> @@ -25,7 +25,7 @@ namespace NBus { TExampleRequest(TAtomic* counterPtr, size_t payloadSize = 320); TExampleRequest(ECreateUninitialized, TAtomic* counterPtr); }; - + class TExampleResponse: public TBusMessage { friend class TExampleProtocol; @@ -37,7 +37,7 @@ namespace NBus { TExampleResponse(TAtomic* counterPtr, size_t payloadSize = 320); TExampleResponse(ECreateUninitialized, TAtomic* counterPtr); }; - + class TExampleProtocol: public TBusProtocol { public: TAtomic RequestCount; @@ -45,7 +45,7 @@ namespace NBus { TAtomic RequestCountDeserialized; TAtomic ResponseCountDeserialized; TAtomic StartCount; - + TExampleProtocol(int port = 0); ~TExampleProtocol() override; @@ -54,28 +54,28 @@ namespace NBus { TAutoPtr<TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) override; }; - + class TExampleClient: private TBusClientHandlerError { public: TExampleProtocol Proto; bool UseCompression; bool CrashOnError; size_t DataSize; - + ssize_t MessageCount; TAtomic RepliesCount; TAtomic Errors; EMessageStatus LastError; - + TSystemEvent WorkDone; - + TBusMessageQueuePtr Bus; TBusClientSessionPtr Session; - + public: TExampleClient(const TBusClientSessionConfig sessionConfig = TBusClientSessionConfig(), int port = 0); ~TExampleClient() override; - + EMessageStatus SendMessage(const TNetAddr* addr = nullptr); void SendMessages(size_t count, const TNetAddr* addr = nullptr); @@ -85,15 +85,15 @@ namespace NBus { void WaitReplies(); EMessageStatus WaitForError(); void WaitForError(EMessageStatus status); - + void SendMessagesWaitReplies(size_t count, const TNetAddr* addr = nullptr); void SendMessagesWaitReplies(size_t count, const TNetAddr& addr); - + void OnReply(TAutoPtr<TBusMessage> mess, TAutoPtr<TBusMessage> reply) override; void OnError(TAutoPtr<TBusMessage> mess, EMessageStatus) override; }; - + class TExampleServer: private TBusServerHandlerError { public: TExampleProtocol Proto; @@ -103,7 +103,7 @@ namespace NBus { bool ForgetRequest; TTestSync TestSync; - + TBusMessageQueuePtr Bus; TBusServerSessionPtr Session; @@ -111,7 +111,7 @@ namespace NBus { TExampleServer( const char* name = "TExampleServer", const TBusServerSessionConfig& sessionConfig = TBusServerSessionConfig()); - + TExampleServer(unsigned port, const char* name = "TExampleServer"); ~TExampleServer() override; @@ -121,9 +121,9 @@ namespace NBus { unsigned GetActualListenPort() const; // any of TNetAddr GetActualListenAddr() const; - + void WaitForOnMessageCount(unsigned n); - + protected: void OnMessage(TOnMessageContext& mess) override; }; diff --git a/library/cpp/messagebus/test/helper/example_module.cpp b/library/cpp/messagebus/test/helper/example_module.cpp index c907825924..65ecfcf73f 100644 --- a/library/cpp/messagebus/test/helper/example_module.cpp +++ b/library/cpp/messagebus/test/helper/example_module.cpp @@ -1,43 +1,43 @@ -#include "example_module.h" - -using namespace NBus; -using namespace NBus::NTest; - -TExampleModule::TExampleModule() - : TBusModule("TExampleModule") -{ - TBusQueueConfig queueConfig; - queueConfig.NumWorkers = 5; - Queue = CreateMessageQueue(queueConfig); -} - -void TExampleModule::StartModule() { - CreatePrivateSessions(Queue.Get()); - StartInput(); -} - -bool TExampleModule::Shutdown() { - TBusModule::Shutdown(); - return true; -} - -TBusServerSessionPtr TExampleModule::CreateExtSession(TBusMessageQueue&) { +#include "example_module.h" + +using namespace NBus; +using namespace NBus::NTest; + +TExampleModule::TExampleModule() + : TBusModule("TExampleModule") +{ + TBusQueueConfig queueConfig; + queueConfig.NumWorkers = 5; + Queue = CreateMessageQueue(queueConfig); +} + +void TExampleModule::StartModule() { + CreatePrivateSessions(Queue.Get()); + StartInput(); +} + +bool TExampleModule::Shutdown() { + TBusModule::Shutdown(); + return true; +} + +TBusServerSessionPtr TExampleModule::CreateExtSession(TBusMessageQueue&) { return nullptr; -} - -TBusServerSessionPtr TExampleServerModule::CreateExtSession(TBusMessageQueue& queue) { - TBusServerSessionPtr r = CreateDefaultDestination(queue, &Proto, TBusServerSessionConfig()); - ServerAddr = TNetAddr("localhost", r->GetActualListenPort()); - return r; -} - +} + +TBusServerSessionPtr TExampleServerModule::CreateExtSession(TBusMessageQueue& queue) { + TBusServerSessionPtr r = CreateDefaultDestination(queue, &Proto, TBusServerSessionConfig()); + ServerAddr = TNetAddr("localhost", r->GetActualListenPort()); + return r; +} + TExampleClientModule::TExampleClientModule() : Source() { } - -TBusServerSessionPtr TExampleClientModule::CreateExtSession(TBusMessageQueue& queue) { - Source = CreateDefaultSource(queue, &Proto, TBusServerSessionConfig()); - Source->RegisterService("localhost"); + +TBusServerSessionPtr TExampleClientModule::CreateExtSession(TBusMessageQueue& queue) { + Source = CreateDefaultSource(queue, &Proto, TBusServerSessionConfig()); + Source->RegisterService("localhost"); return nullptr; -} +} diff --git a/library/cpp/messagebus/test/helper/example_module.h b/library/cpp/messagebus/test/helper/example_module.h index b1b0a6dd14..a0b295f613 100644 --- a/library/cpp/messagebus/test/helper/example_module.h +++ b/library/cpp/messagebus/test/helper/example_module.h @@ -1,7 +1,7 @@ -#pragma once - -#include "example.h" - +#pragma once + +#include "example.h" + #include <library/cpp/messagebus/oldmodule/module.h> namespace NBus { @@ -9,29 +9,29 @@ namespace NBus { struct TExampleModule: public TBusModule { TExampleProtocol Proto; TBusMessageQueuePtr Queue; - + TExampleModule(); - + void StartModule(); - + bool Shutdown() override; - + // nop by default TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override; }; - + struct TExampleServerModule: public TExampleModule { TNetAddr ServerAddr; TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override; }; - + struct TExampleClientModule: public TExampleModule { TBusClientSessionPtr Source; - + TExampleClientModule(); - + TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override; }; - + } } diff --git a/library/cpp/messagebus/test/helper/fixed_port.cpp b/library/cpp/messagebus/test/helper/fixed_port.cpp index f83ce3161a..258da0d1a5 100644 --- a/library/cpp/messagebus/test/helper/fixed_port.cpp +++ b/library/cpp/messagebus/test/helper/fixed_port.cpp @@ -1,10 +1,10 @@ #include "fixed_port.h" #include <util/system/env.h> - + #include <stdlib.h> - + bool NBus::NTest::IsFixedPortTestAllowed() { - // TODO: report skipped tests to test + // TODO: report skipped tests to test return !GetEnv("MB_TESTS_SKIP_FIXED_PORT"); -} +} diff --git a/library/cpp/messagebus/test/helper/fixed_port.h b/library/cpp/messagebus/test/helper/fixed_port.h index 39c8da4dfa..a9c61ebc63 100644 --- a/library/cpp/messagebus/test/helper/fixed_port.h +++ b/library/cpp/messagebus/test/helper/fixed_port.h @@ -1,11 +1,11 @@ -#pragma once - +#pragma once + namespace NBus { namespace NTest { bool IsFixedPortTestAllowed(); - + // Must not be in range OS uses for bind on random port. const unsigned FixedPort = 4927; - + } } diff --git a/library/cpp/messagebus/test/helper/hanging_server.cpp b/library/cpp/messagebus/test/helper/hanging_server.cpp index 3911ff10ba..a35514b00d 100644 --- a/library/cpp/messagebus/test/helper/hanging_server.cpp +++ b/library/cpp/messagebus/test/helper/hanging_server.cpp @@ -1,13 +1,13 @@ #include "hanging_server.h" -#include <util/system/yassert.h> - -using namespace NBus; - +#include <util/system/yassert.h> + +using namespace NBus; + THangingServer::THangingServer(int port) { BindResult = BindOnPort(port, false); -} - -int THangingServer::GetPort() const { - return BindResult.first; -} +} + +int THangingServer::GetPort() const { + return BindResult.first; +} diff --git a/library/cpp/messagebus/test/helper/hanging_server.h b/library/cpp/messagebus/test/helper/hanging_server.h index 384f07d7cf..cc9fb274d8 100644 --- a/library/cpp/messagebus/test/helper/hanging_server.h +++ b/library/cpp/messagebus/test/helper/hanging_server.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/network.h> -#include <util/network/sock.h> - -class THangingServer { -private: +#include <util/network/sock.h> + +class THangingServer { +private: std::pair<unsigned, TVector<NBus::TBindResult>> BindResult; -public: - // listen on given port, and nothing else - THangingServer(int port = 0); - // actual port - int GetPort() const; -}; +public: + // listen on given port, and nothing else + THangingServer(int port = 0); + // actual port + int GetPort() const; +}; diff --git a/library/cpp/messagebus/test/helper/message_handler_error.cpp b/library/cpp/messagebus/test/helper/message_handler_error.cpp index 40997adec8..c09811ec67 100644 --- a/library/cpp/messagebus/test/helper/message_handler_error.cpp +++ b/library/cpp/messagebus/test/helper/message_handler_error.cpp @@ -1,26 +1,26 @@ #include "message_handler_error.h" -#include <util/system/yassert.h> - -using namespace NBus; -using namespace NBus::NTest; - -void TBusClientHandlerError::OnError(TAutoPtr<TBusMessage>, EMessageStatus status) { +#include <util/system/yassert.h> + +using namespace NBus; +using namespace NBus::NTest; + +void TBusClientHandlerError::OnError(TAutoPtr<TBusMessage>, EMessageStatus status) { Y_FAIL("must not be called, status: %s", ToString(status).data()); -} - -void TBusClientHandlerError::OnReply(TAutoPtr<TBusMessage>, TAutoPtr<TBusMessage>) { +} + +void TBusClientHandlerError::OnReply(TAutoPtr<TBusMessage>, TAutoPtr<TBusMessage>) { Y_FAIL("must not be called"); -} - -void TBusClientHandlerError::OnMessageSentOneWay(TAutoPtr<TBusMessage>) { +} + +void TBusClientHandlerError::OnMessageSentOneWay(TAutoPtr<TBusMessage>) { Y_FAIL("must not be called"); -} - -void TBusServerHandlerError::OnError(TAutoPtr<TBusMessage>, EMessageStatus status) { +} + +void TBusServerHandlerError::OnError(TAutoPtr<TBusMessage>, EMessageStatus status) { Y_FAIL("must not be called, status: %s", ToString(status).data()); -} - -void TBusServerHandlerError::OnMessage(TOnMessageContext&) { +} + +void TBusServerHandlerError::OnMessage(TOnMessageContext&) { Y_FAIL("must not be called"); -} +} diff --git a/library/cpp/messagebus/test/helper/message_handler_error.h b/library/cpp/messagebus/test/helper/message_handler_error.h index bba0007a44..a314b10761 100644 --- a/library/cpp/messagebus/test/helper/message_handler_error.h +++ b/library/cpp/messagebus/test/helper/message_handler_error.h @@ -1,7 +1,7 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/ybus.h> - + namespace NBus { namespace NTest { struct TBusClientHandlerError: public IBusClientHandler { @@ -9,11 +9,11 @@ namespace NBus { void OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage) override; void OnReply(TAutoPtr<TBusMessage> pMessage, TAutoPtr<TBusMessage> pReply) override; }; - + struct TBusServerHandlerError: public IBusServerHandler { void OnError(TAutoPtr<TBusMessage> pMessage, EMessageStatus status) override; void OnMessage(TOnMessageContext& pMessage) override; }; - + } } diff --git a/library/cpp/messagebus/test/helper/object_count_check.h b/library/cpp/messagebus/test/helper/object_count_check.h index d844469fb9..1c4756e58c 100644 --- a/library/cpp/messagebus/test/helper/object_count_check.h +++ b/library/cpp/messagebus/test/helper/object_count_check.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/messagebus/remote_client_connection.h> @@ -9,66 +9,66 @@ #include <library/cpp/messagebus/ybus.h> #include <library/cpp/messagebus/oldmodule/module.h> #include <library/cpp/messagebus/scheduler/scheduler.h> - + #include <util/generic/object_counter.h> #include <util/system/type_name.h> #include <util/stream/output.h> - + #include <typeinfo> -struct TObjectCountCheck { - bool Enabled; - - template <typename T> - struct TReset { - TObjectCountCheck* const Thiz; - +struct TObjectCountCheck { + bool Enabled; + + template <typename T> + struct TReset { + TObjectCountCheck* const Thiz; + TReset(TObjectCountCheck* thiz) : Thiz(thiz) { } - + void operator()() { long oldValue = TObjectCounter<T>::ResetObjectCount(); - if (oldValue != 0) { + if (oldValue != 0) { Cerr << "warning: previous counter: " << oldValue << " for " << TypeName<T>() << Endl; - Cerr << "won't check in this test" << Endl; - Thiz->Enabled = false; - } - } - }; - - TObjectCountCheck() { - Enabled = true; - DoForAllCounters<TReset>(); - } - - template <typename T> - struct TCheckZero { + Cerr << "won't check in this test" << Endl; + Thiz->Enabled = false; + } + } + }; + + TObjectCountCheck() { + Enabled = true; + DoForAllCounters<TReset>(); + } + + template <typename T> + struct TCheckZero { TCheckZero(TObjectCountCheck*) { } - + void operator()() { UNIT_ASSERT_VALUES_EQUAL_C(0L, TObjectCounter<T>::ObjectCount(), TypeName<T>()); - } - }; - - ~TObjectCountCheck() { - if (Enabled) { - DoForAllCounters<TCheckZero>(); - } - } - - template <template <typename> class TOp> - void DoForAllCounters() { - TOp< ::NBus::NPrivate::TRemoteClientConnection>(this)(); - TOp< ::NBus::NPrivate::TRemoteServerConnection>(this)(); - TOp< ::NBus::NPrivate::TRemoteClientSession>(this)(); - TOp< ::NBus::NPrivate::TRemoteServerSession>(this)(); - TOp< ::NBus::NPrivate::TScheduler>(this)(); - TOp< ::NEventLoop::TEventLoop>(this)(); - TOp< ::NEventLoop::TChannel>(this)(); - TOp< ::NBus::TBusModule>(this)(); - TOp< ::NBus::TBusJob>(this)(); - } -}; + } + }; + + ~TObjectCountCheck() { + if (Enabled) { + DoForAllCounters<TCheckZero>(); + } + } + + template <template <typename> class TOp> + void DoForAllCounters() { + TOp< ::NBus::NPrivate::TRemoteClientConnection>(this)(); + TOp< ::NBus::NPrivate::TRemoteServerConnection>(this)(); + TOp< ::NBus::NPrivate::TRemoteClientSession>(this)(); + TOp< ::NBus::NPrivate::TRemoteServerSession>(this)(); + TOp< ::NBus::NPrivate::TScheduler>(this)(); + TOp< ::NEventLoop::TEventLoop>(this)(); + TOp< ::NEventLoop::TChannel>(this)(); + TOp< ::NBus::TBusModule>(this)(); + TOp< ::NBus::TBusJob>(this)(); + } +}; diff --git a/library/cpp/messagebus/test/helper/wait_for.h b/library/cpp/messagebus/test/helper/wait_for.h index ebd0bfd6e2..f09958d4c0 100644 --- a/library/cpp/messagebus/test/helper/wait_for.h +++ b/library/cpp/messagebus/test/helper/wait_for.h @@ -1,14 +1,14 @@ -#pragma once - -#include <util/datetime/base.h> -#include <util/system/yassert.h> - +#pragma once + +#include <util/datetime/base.h> +#include <util/system/yassert.h> + #define UNIT_WAIT_FOR(condition) \ do { \ - TInstant start(TInstant::Now()); \ - while (!(condition) && (TInstant::Now() - start < TDuration::Seconds(10))) { \ + TInstant start(TInstant::Now()); \ + while (!(condition) && (TInstant::Now() - start < TDuration::Seconds(10))) { \ Sleep(TDuration::MilliSeconds(1)); \ - } \ - /* TODO: use UNIT_ASSERT if in unittest thread */ \ + } \ + /* TODO: use UNIT_ASSERT if in unittest thread */ \ Y_VERIFY(condition, "condition failed after 10 seconds wait"); \ - } while (0) + } while (0) diff --git a/library/cpp/messagebus/test/helper/ya.make b/library/cpp/messagebus/test/helper/ya.make index 703f6b6953..97bd45f573 100644 --- a/library/cpp/messagebus/test/helper/ya.make +++ b/library/cpp/messagebus/test/helper/ya.make @@ -1,17 +1,17 @@ -LIBRARY(messagebus_test_helper) - +LIBRARY(messagebus_test_helper) + OWNER(g:messagebus) - -SRCS( - example.cpp - example_module.cpp - fixed_port.cpp - message_handler_error.cpp - hanging_server.cpp -) - -PEERDIR( + +SRCS( + example.cpp + example_module.cpp + fixed_port.cpp + message_handler_error.cpp + hanging_server.cpp +) + +PEERDIR( library/cpp/messagebus/oldmodule -) - -END() +) + +END() diff --git a/library/cpp/messagebus/test/perftest/messages.proto b/library/cpp/messagebus/test/perftest/messages.proto index a48bb2f480..8919034e7a 100644 --- a/library/cpp/messagebus/test/perftest/messages.proto +++ b/library/cpp/messagebus/test/perftest/messages.proto @@ -1,7 +1,7 @@ -message TPerftestRequestRecord { - required string Data = 1; -} - -message TPerftestResponseRecord { - required string Data = 1; -} +message TPerftestRequestRecord { + required string Data = 1; +} + +message TPerftestResponseRecord { + required string Data = 1; +} diff --git a/library/cpp/messagebus/test/perftest/perftest.cpp b/library/cpp/messagebus/test/perftest/perftest.cpp index 44fb4d837d..8489319278 100644 --- a/library/cpp/messagebus/test/perftest/perftest.cpp +++ b/library/cpp/messagebus/test/perftest/perftest.cpp @@ -15,10 +15,10 @@ #include <library/cpp/lwtrace/start.h> #include <library/cpp/sighandler/async_signals_handler.h> #include <library/cpp/threading/future/legacy_future.h> - + #include <util/generic/ptr.h> #include <util/generic/string.h> -#include <util/generic/vector.h> +#include <util/generic/vector.h> #include <util/generic/yexception.h> #include <util/random/random.h> #include <util/stream/file.h> @@ -29,18 +29,18 @@ #include <util/system/sysstat.h> #include <util/system/thread.h> #include <util/thread/lfqueue.h> - + #include <signal.h> #include <stdlib.h> - -using namespace NBus; - -/////////////////////////////////////////////////////// -/// \brief Configuration parameters of the test - -const int DEFAULT_PORT = 55666; - -struct TPerftestConfig { + +using namespace NBus; + +/////////////////////////////////////////////////////// +/// \brief Configuration parameters of the test + +const int DEFAULT_PORT = 55666; + +struct TPerftestConfig { TString Nodes; ///< node1:port1,node2:port2 int ClientCount; int MessageSize; ///< size of message to send @@ -53,144 +53,144 @@ struct TPerftestConfig { bool ExecuteOnReplyInWorkerPool; bool UseCompression; bool Profile; - unsigned WwwPort; - - TPerftestConfig(); - - void Print() { - fprintf(stderr, "ClientCount=%d\n", ClientCount); - fprintf(stderr, "ServerPort=%d\n", ServerPort); - fprintf(stderr, "Delay=%d usecs\n", Delay); + unsigned WwwPort; + + TPerftestConfig(); + + void Print() { + fprintf(stderr, "ClientCount=%d\n", ClientCount); + fprintf(stderr, "ServerPort=%d\n", ServerPort); + fprintf(stderr, "Delay=%d usecs\n", Delay); fprintf(stderr, "MessageSize=%d bytes\n", MessageSize); fprintf(stderr, "Failure=%.3f%%\n", Failure * 100.0); - fprintf(stderr, "Runtime=%d seconds\n", Run); - fprintf(stderr, "ServerUseModules=%s\n", ServerUseModules ? "true" : "false"); - fprintf(stderr, "ExecuteOnMessageInWorkerPool=%s\n", ExecuteOnMessageInWorkerPool ? "true" : "false"); - fprintf(stderr, "ExecuteOnReplyInWorkerPool=%s\n", ExecuteOnReplyInWorkerPool ? "true" : "false"); - fprintf(stderr, "UseCompression=%s\n", UseCompression ? "true" : "false"); - fprintf(stderr, "Profile=%s\n", Profile ? "true" : "false"); - fprintf(stderr, "WwwPort=%u\n", WwwPort); - } -}; - + fprintf(stderr, "Runtime=%d seconds\n", Run); + fprintf(stderr, "ServerUseModules=%s\n", ServerUseModules ? "true" : "false"); + fprintf(stderr, "ExecuteOnMessageInWorkerPool=%s\n", ExecuteOnMessageInWorkerPool ? "true" : "false"); + fprintf(stderr, "ExecuteOnReplyInWorkerPool=%s\n", ExecuteOnReplyInWorkerPool ? "true" : "false"); + fprintf(stderr, "UseCompression=%s\n", UseCompression ? "true" : "false"); + fprintf(stderr, "Profile=%s\n", Profile ? "true" : "false"); + fprintf(stderr, "WwwPort=%u\n", WwwPort); + } +}; + extern TPerftestConfig* TheConfig; -extern bool TheExit; - +extern bool TheExit; + TVector<TNetAddr> ServerAddresses; - -struct TConfig { - TBusQueueConfig ServerQueueConfig; - TBusQueueConfig ClientQueueConfig; - TBusServerSessionConfig ServerSessionConfig; - TBusClientSessionConfig ClientSessionConfig; - bool SimpleProtocol; - -private: - void ConfigureDefaults(TBusQueueConfig& config) { - config.NumWorkers = 4; - } - - void ConfigureDefaults(TBusSessionConfig& config) { - config.MaxInFlight = 10000; - config.SendTimeout = TDuration::Seconds(20).MilliSeconds(); - config.TotalTimeout = TDuration::Seconds(60).MilliSeconds(); - } - -public: - TConfig() - : SimpleProtocol(false) - { - ConfigureDefaults(ServerQueueConfig); - ConfigureDefaults(ClientQueueConfig); - ConfigureDefaults(ServerSessionConfig); - ConfigureDefaults(ClientSessionConfig); - } - - void Print() { - // TODO: do not print server if only client and vice verse - Cerr << "server queue config:\n"; - Cerr << IndentText(ServerQueueConfig.PrintToString()); - Cerr << "server session config:" << Endl; - Cerr << IndentText(ServerSessionConfig.PrintToString()); - Cerr << "client queue config:\n"; - Cerr << IndentText(ClientQueueConfig.PrintToString()); - Cerr << "client session config:" << Endl; - Cerr << IndentText(ClientSessionConfig.PrintToString()); - Cerr << "simple protocol: " << SimpleProtocol << "\n"; - } -}; - -TConfig Config; - -//////////////////////////////////////////////////////////////// -/// \brief Fast message - + +struct TConfig { + TBusQueueConfig ServerQueueConfig; + TBusQueueConfig ClientQueueConfig; + TBusServerSessionConfig ServerSessionConfig; + TBusClientSessionConfig ClientSessionConfig; + bool SimpleProtocol; + +private: + void ConfigureDefaults(TBusQueueConfig& config) { + config.NumWorkers = 4; + } + + void ConfigureDefaults(TBusSessionConfig& config) { + config.MaxInFlight = 10000; + config.SendTimeout = TDuration::Seconds(20).MilliSeconds(); + config.TotalTimeout = TDuration::Seconds(60).MilliSeconds(); + } + +public: + TConfig() + : SimpleProtocol(false) + { + ConfigureDefaults(ServerQueueConfig); + ConfigureDefaults(ClientQueueConfig); + ConfigureDefaults(ServerSessionConfig); + ConfigureDefaults(ClientSessionConfig); + } + + void Print() { + // TODO: do not print server if only client and vice verse + Cerr << "server queue config:\n"; + Cerr << IndentText(ServerQueueConfig.PrintToString()); + Cerr << "server session config:" << Endl; + Cerr << IndentText(ServerSessionConfig.PrintToString()); + Cerr << "client queue config:\n"; + Cerr << IndentText(ClientQueueConfig.PrintToString()); + Cerr << "client session config:" << Endl; + Cerr << IndentText(ClientSessionConfig.PrintToString()); + Cerr << "simple protocol: " << SimpleProtocol << "\n"; + } +}; + +TConfig Config; + +//////////////////////////////////////////////////////////////// +/// \brief Fast message + using TPerftestRequest = TBusBufferMessage<TPerftestRequestRecord, 77>; using TPerftestResponse = TBusBufferMessage<TPerftestResponseRecord, 79>; - -static size_t RequestSize() { - return RandomNumber<size_t>(TheConfig->MessageSize * 2 + 1); -} - -TAutoPtr<TBusMessage> NewRequest() { - if (Config.SimpleProtocol) { - TAutoPtr<TSimpleMessage> r(new TSimpleMessage); - r->SetCompressed(TheConfig->UseCompression); - r->Payload = 10; - return r.Release(); - } else { - TAutoPtr<TPerftestRequest> r(new TPerftestRequest); - r->SetCompressed(TheConfig->UseCompression); - // TODO: use random content for better compression test + +static size_t RequestSize() { + return RandomNumber<size_t>(TheConfig->MessageSize * 2 + 1); +} + +TAutoPtr<TBusMessage> NewRequest() { + if (Config.SimpleProtocol) { + TAutoPtr<TSimpleMessage> r(new TSimpleMessage); + r->SetCompressed(TheConfig->UseCompression); + r->Payload = 10; + return r.Release(); + } else { + TAutoPtr<TPerftestRequest> r(new TPerftestRequest); + r->SetCompressed(TheConfig->UseCompression); + // TODO: use random content for better compression test r->Record.SetData(TString(RequestSize(), '?')); - return r.Release(); - } -} - -void CheckRequest(TPerftestRequest* request) { + return r.Release(); + } +} + +void CheckRequest(TPerftestRequest* request) { const TString& data = request->Record.GetData(); - for (size_t i = 0; i != data.size(); ++i) { + for (size_t i = 0; i != data.size(); ++i) { Y_VERIFY(data.at(i) == '?', "must be question mark"); - } -} - -TAutoPtr<TPerftestResponse> NewResponse(TPerftestRequest* request) { - TAutoPtr<TPerftestResponse> r(new TPerftestResponse); - r->SetCompressed(TheConfig->UseCompression); + } +} + +TAutoPtr<TPerftestResponse> NewResponse(TPerftestRequest* request) { + TAutoPtr<TPerftestResponse> r(new TPerftestResponse); + r->SetCompressed(TheConfig->UseCompression); r->Record.SetData(TString(request->Record.GetData().size(), '.')); - return r; -} - -void CheckResponse(TPerftestResponse* response) { + return r; +} + +void CheckResponse(TPerftestResponse* response) { const TString& data = response->Record.GetData(); - for (size_t i = 0; i != data.size(); ++i) { + for (size_t i = 0; i != data.size(); ++i) { Y_VERIFY(data.at(i) == '.', "must be dot"); - } -} - -//////////////////////////////////////////////////////////////////// -/// \brief Fast protocol that common between client and server -class TPerftestProtocol: public TBusBufferProtocol { -public: - TPerftestProtocol() - : TBusBufferProtocol("TPerftestProtocol", TheConfig->ServerPort) - { - RegisterType(new TPerftestRequest); - RegisterType(new TPerftestResponse); - } -}; - -class TPerftestServer; -class TPerftestUsingModule; -class TPerftestClient; - -struct TTestStats { - TInstant Start; - - TAtomic Messages; - TAtomic Errors; - TAtomic Replies; - + } +} + +//////////////////////////////////////////////////////////////////// +/// \brief Fast protocol that common between client and server +class TPerftestProtocol: public TBusBufferProtocol { +public: + TPerftestProtocol() + : TBusBufferProtocol("TPerftestProtocol", TheConfig->ServerPort) + { + RegisterType(new TPerftestRequest); + RegisterType(new TPerftestResponse); + } +}; + +class TPerftestServer; +class TPerftestUsingModule; +class TPerftestClient; + +struct TTestStats { + TInstant Start; + + TAtomic Messages; + TAtomic Errors; + TAtomic Replies; + void IncMessage() { AtomicIncrement(Messages); } @@ -211,265 +211,265 @@ struct TTestStats { int NumReplies() { return AtomicGet(Replies); } - + double GetThroughput() { - return NumReplies() * 1000000.0 / (TInstant::Now() - Start).MicroSeconds(); - } - -public: - TTestStats() - : Start(TInstant::Now()) - , Messages(0) - , Errors(0) - , Replies(0) - { - } - - void PeriodicallyPrint(); -}; - -TTestStats Stats; - -//////////////////////////////////////////////////////////////////// -/// \brief Fast of the client session + return NumReplies() * 1000000.0 / (TInstant::Now() - Start).MicroSeconds(); + } + +public: + TTestStats() + : Start(TInstant::Now()) + , Messages(0) + , Errors(0) + , Replies(0) + { + } + + void PeriodicallyPrint(); +}; + +TTestStats Stats; + +//////////////////////////////////////////////////////////////////// +/// \brief Fast of the client session class TPerftestClient : IBusClientHandler { -public: - TBusClientSessionPtr Session; - THolder<TBusProtocol> Proto; - TBusMessageQueuePtr Bus; +public: + TBusClientSessionPtr Session; + THolder<TBusProtocol> Proto; + TBusMessageQueuePtr Bus; TVector<TBusClientConnectionPtr> Connections; - -public: - /// constructor creates instances of protocol and session - TPerftestClient() { - /// create or get instance of message queue, need one per application - Bus = CreateMessageQueue(Config.ClientQueueConfig, "client"); - - if (Config.SimpleProtocol) { - Proto.Reset(new TSimpleProtocol); - } else { - Proto.Reset(new TPerftestProtocol); - } - - Session = TBusClientSession::Create(Proto.Get(), this, Config.ClientSessionConfig, Bus); - - for (unsigned i = 0; i < ServerAddresses.size(); ++i) { - Connections.push_back(Session->GetConnection(ServerAddresses[i])); - } - } - - /// dispatch of requests is done here - void Work() { + +public: + /// constructor creates instances of protocol and session + TPerftestClient() { + /// create or get instance of message queue, need one per application + Bus = CreateMessageQueue(Config.ClientQueueConfig, "client"); + + if (Config.SimpleProtocol) { + Proto.Reset(new TSimpleProtocol); + } else { + Proto.Reset(new TPerftestProtocol); + } + + Session = TBusClientSession::Create(Proto.Get(), this, Config.ClientSessionConfig, Bus); + + for (unsigned i = 0; i < ServerAddresses.size(); ++i) { + Connections.push_back(Session->GetConnection(ServerAddresses[i])); + } + } + + /// dispatch of requests is done here + void Work() { SetCurrentThreadName("FastClient::Work"); - - while (!TheExit) { - TBusClientConnection* connection; - if (Connections.size() == 1) { - connection = Connections.front().Get(); - } else { - connection = Connections.at(RandomNumber<size_t>()).Get(); - } - + + while (!TheExit) { + TBusClientConnection* connection; + if (Connections.size() == 1) { + connection = Connections.front().Get(); + } else { + connection = Connections.at(RandomNumber<size_t>()).Get(); + } + TBusMessage* message = NewRequest().Release(); - int ret = connection->SendMessage(message, true); - - if (ret == MESSAGE_OK) { - Stats.IncMessage(); - } else if (ret == MESSAGE_BUSY) { - //delete message; - //Sleep(TDuration::MilliSeconds(1)); - //continue; + int ret = connection->SendMessage(message, true); + + if (ret == MESSAGE_OK) { + Stats.IncMessage(); + } else if (ret == MESSAGE_BUSY) { + //delete message; + //Sleep(TDuration::MilliSeconds(1)); + //continue; Y_FAIL("unreachable"); - } else if (ret == MESSAGE_SHUTDOWN) { - delete message; - } else { - delete message; - Stats.IncErrors(); - } - } - } - - void Stop() { - Session->Shutdown(); - } - - /// actual work is being done here + } else if (ret == MESSAGE_SHUTDOWN) { + delete message; + } else { + delete message; + Stats.IncErrors(); + } + } + } + + void Stop() { + Session->Shutdown(); + } + + /// actual work is being done here void OnReply(TAutoPtr<TBusMessage> mess, TAutoPtr<TBusMessage> reply) override { Y_UNUSED(mess); - - if (Config.SimpleProtocol) { - VerifyDynamicCast<TSimpleMessage*>(reply.Get()); - } else { - TPerftestResponse* typed = VerifyDynamicCast<TPerftestResponse*>(reply.Get()); - - CheckResponse(typed); - } - - Stats.IncReplies(); - } - - /// message that could not be delivered + + if (Config.SimpleProtocol) { + VerifyDynamicCast<TSimpleMessage*>(reply.Get()); + } else { + TPerftestResponse* typed = VerifyDynamicCast<TPerftestResponse*>(reply.Get()); + + CheckResponse(typed); + } + + Stats.IncReplies(); + } + + /// message that could not be delivered void OnError(TAutoPtr<TBusMessage> mess, EMessageStatus status) override { Y_UNUSED(mess); Y_UNUSED(status); - - if (TheExit) { - return; - } - - Stats.IncErrors(); - + + if (TheExit) { + return; + } + + Stats.IncErrors(); + // Y_ASSERT(TheConfig->Failure > 0.0); - } -}; - -class TPerftestServerCommon { -public: - THolder<TBusProtocol> Proto; - - TBusMessageQueuePtr Bus; - - TBusServerSessionPtr Session; - -protected: - TPerftestServerCommon(const char* name) - : Session() - { - if (Config.SimpleProtocol) { - Proto.Reset(new TSimpleProtocol); - } else { - Proto.Reset(new TPerftestProtocol); - } - - /// create or get instance of single message queue, need one for application - Bus = CreateMessageQueue(Config.ServerQueueConfig, name); - } - -public: - void Stop() { - Session->Shutdown(); - } -}; - -struct TAsyncRequest { - TBusMessage* Request; - TInstant ReceivedTime; -}; - -///////////////////////////////////////////////////////////////////// -/// \brief Fast of the server session -class TPerftestServer: public TPerftestServerCommon, public IBusServerHandler { -public: - TLockFreeQueue<TAsyncRequest> AsyncRequests; - -public: - TPerftestServer() - : TPerftestServerCommon("server") - { - /// register destination session - Session = TBusServerSession::Create(Proto.Get(), this, Config.ServerSessionConfig, Bus); + } +}; + +class TPerftestServerCommon { +public: + THolder<TBusProtocol> Proto; + + TBusMessageQueuePtr Bus; + + TBusServerSessionPtr Session; + +protected: + TPerftestServerCommon(const char* name) + : Session() + { + if (Config.SimpleProtocol) { + Proto.Reset(new TSimpleProtocol); + } else { + Proto.Reset(new TPerftestProtocol); + } + + /// create or get instance of single message queue, need one for application + Bus = CreateMessageQueue(Config.ServerQueueConfig, name); + } + +public: + void Stop() { + Session->Shutdown(); + } +}; + +struct TAsyncRequest { + TBusMessage* Request; + TInstant ReceivedTime; +}; + +///////////////////////////////////////////////////////////////////// +/// \brief Fast of the server session +class TPerftestServer: public TPerftestServerCommon, public IBusServerHandler { +public: + TLockFreeQueue<TAsyncRequest> AsyncRequests; + +public: + TPerftestServer() + : TPerftestServerCommon("server") + { + /// register destination session + Session = TBusServerSession::Create(Proto.Get(), this, Config.ServerSessionConfig, Bus); Y_ASSERT(Session && "probably somebody is listening on the same port"); - } - - /// when message comes, send reply + } + + /// when message comes, send reply void OnMessage(TOnMessageContext& mess) override { - if (Config.SimpleProtocol) { - TSimpleMessage* typed = VerifyDynamicCast<TSimpleMessage*>(mess.GetMessage()); - TAutoPtr<TSimpleMessage> response(new TSimpleMessage); - response->Payload = typed->Payload; - mess.SendReplyMove(response); - return; - } - - TPerftestRequest* typed = VerifyDynamicCast<TPerftestRequest*>(mess.GetMessage()); - - CheckRequest(typed); - - /// forget replies for few messages, see what happends + if (Config.SimpleProtocol) { + TSimpleMessage* typed = VerifyDynamicCast<TSimpleMessage*>(mess.GetMessage()); + TAutoPtr<TSimpleMessage> response(new TSimpleMessage); + response->Payload = typed->Payload; + mess.SendReplyMove(response); + return; + } + + TPerftestRequest* typed = VerifyDynamicCast<TPerftestRequest*>(mess.GetMessage()); + + CheckRequest(typed); + + /// forget replies for few messages, see what happends if (TheConfig->Failure > RandomNumber<double>()) { - return; - } - - /// sleep requested time - if (TheConfig->Delay) { - TAsyncRequest request; - request.Request = mess.ReleaseMessage(); - request.ReceivedTime = TInstant::Now(); - AsyncRequests.Enqueue(request); - return; - } - - TAutoPtr<TPerftestResponse> reply(NewResponse(typed)); - /// sent empty reply for each message - mess.SendReplyMove(reply); - // TODO: count results - } - - void Stop() { - TPerftestServerCommon::Stop(); - } -}; - -class TPerftestUsingModule: public TPerftestServerCommon, public TBusModule { -public: - TPerftestUsingModule() - : TPerftestServerCommon("server") - , TBusModule("fast") - { + return; + } + + /// sleep requested time + if (TheConfig->Delay) { + TAsyncRequest request; + request.Request = mess.ReleaseMessage(); + request.ReceivedTime = TInstant::Now(); + AsyncRequests.Enqueue(request); + return; + } + + TAutoPtr<TPerftestResponse> reply(NewResponse(typed)); + /// sent empty reply for each message + mess.SendReplyMove(reply); + // TODO: count results + } + + void Stop() { + TPerftestServerCommon::Stop(); + } +}; + +class TPerftestUsingModule: public TPerftestServerCommon, public TBusModule { +public: + TPerftestUsingModule() + : TPerftestServerCommon("server") + , TBusModule("fast") + { Y_VERIFY(CreatePrivateSessions(Bus.Get()), "failed to initialize dupdetect module"); Y_VERIFY(StartInput(), "failed to start input"); - } - + } + ~TPerftestUsingModule() override { - Shutdown(); - } - -private: + Shutdown(); + } + +private: TJobHandler Start(TBusJob* job, TBusMessage* mess) override { - TPerftestRequest* typed = VerifyDynamicCast<TPerftestRequest*>(mess); - CheckRequest(typed); - - /// sleep requested time - if (TheConfig->Delay) { - usleep(TheConfig->Delay); - } - - /// forget replies for few messages, see what happends + TPerftestRequest* typed = VerifyDynamicCast<TPerftestRequest*>(mess); + CheckRequest(typed); + + /// sleep requested time + if (TheConfig->Delay) { + usleep(TheConfig->Delay); + } + + /// forget replies for few messages, see what happends if (TheConfig->Failure > RandomNumber<double>()) { return nullptr; - } - - job->SendReply(NewResponse(typed).Release()); + } + + job->SendReply(NewResponse(typed).Release()); return nullptr; - } - + } + TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override { - return Session = CreateDefaultDestination(queue, Proto.Get(), Config.ServerSessionConfig); - } -}; - + return Session = CreateDefaultDestination(queue, Proto.Get(), Config.ServerSessionConfig); + } +}; + // ./perftest/perftest -s 11456 -c localhost:11456 -r 60 -n 4 -i 5000 using namespace std; using namespace NBus; -static TNetworkAddress ParseNetworkAddress(const char* string) { +static TNetworkAddress ParseNetworkAddress(const char* string) { TString Name; int Port; const char* port = strchr(string, ':'); if (port != nullptr) { - Name.append(string, port - string); + Name.append(string, port - string); Port = atoi(port + 1); } else { - Name.append(string); - Port = TheConfig->ServerPort != 0 ? TheConfig->ServerPort : DEFAULT_PORT; + Name.append(string); + Port = TheConfig->ServerPort != 0 ? TheConfig->ServerPort : DEFAULT_PORT; } - return TNetworkAddress(Name, Port); -} - + return TNetworkAddress(Name, Port); +} + TVector<TNetAddr> ParseNodes(const TString nodes) { TVector<TNetAddr> r; @@ -480,234 +480,234 @@ TVector<TNetAddr> ParseNodes(const TString nodes) { for (int i = 0; i < int(numh); i++) { const TNetworkAddress& networkAddress = ParseNetworkAddress(hosts[i].data()); Y_VERIFY(networkAddress.Begin() != networkAddress.End(), "no addresses"); - r.push_back(TNetAddr(networkAddress, &*networkAddress.Begin())); + r.push_back(TNetAddr(networkAddress, &*networkAddress.Begin())); } - return r; + return r; } -TPerftestConfig::TPerftestConfig() { - TBusSessionConfig defaultConfig; - - ServerPort = DEFAULT_PORT; +TPerftestConfig::TPerftestConfig() { + TBusSessionConfig defaultConfig; + + ServerPort = DEFAULT_PORT; Delay = 0; // artificial delay inside server OnMessage() MessageSize = 200; Failure = 0.00; Run = 60; // in seconds Nodes = "localhost"; - ServerUseModules = false; - ExecuteOnMessageInWorkerPool = defaultConfig.ExecuteOnMessageInWorkerPool; - ExecuteOnReplyInWorkerPool = defaultConfig.ExecuteOnReplyInWorkerPool; - UseCompression = false; - Profile = false; - WwwPort = 0; + ServerUseModules = false; + ExecuteOnMessageInWorkerPool = defaultConfig.ExecuteOnMessageInWorkerPool; + ExecuteOnReplyInWorkerPool = defaultConfig.ExecuteOnReplyInWorkerPool; + UseCompression = false; + Profile = false; + WwwPort = 0; } TPerftestConfig* TheConfig = new TPerftestConfig(); bool TheExit = false; - + TSystemEvent StopEvent; TSimpleSharedPtr<TPerftestServer> Server; TSimpleSharedPtr<TPerftestUsingModule> ServerUsingModule; - + TVector<TSimpleSharedPtr<TPerftestClient>> Clients; -TMutex ClientsLock; - +TMutex ClientsLock; + void stopsignal(int /*sig*/) { fprintf(stderr, "\n-------------------- exiting ------------------\n"); TheExit = true; - StopEvent.Signal(); + StopEvent.Signal(); } // -s <num> - start server on port <num> // -c <node:port,node:port> - start client -void TTestStats::PeriodicallyPrint() { +void TTestStats::PeriodicallyPrint() { SetCurrentThreadName("print-stats"); - - for (;;) { - StopEvent.WaitT(TDuration::Seconds(1)); - if (TheExit) - break; - + + for (;;) { + StopEvent.WaitT(TDuration::Seconds(1)); + if (TheExit) + break; + TVector<TSimpleSharedPtr<TPerftestClient>> clients; - { - TGuard<TMutex> guard(ClientsLock); - clients = Clients; - } - - fprintf(stderr, "replies=%d errors=%d throughput=%.3f mess/sec\n", + { + TGuard<TMutex> guard(ClientsLock); + clients = Clients; + } + + fprintf(stderr, "replies=%d errors=%d throughput=%.3f mess/sec\n", NumReplies(), NumErrors(), GetThroughput()); - if (!!Server) { - fprintf(stderr, "server: q: %u %s\n", + if (!!Server) { + fprintf(stderr, "server: q: %u %s\n", (unsigned)Server->Bus->GetExecutor()->GetWorkQueueSize(), Server->Session->GetStatusSingleLine().data()); - } - if (!!ServerUsingModule) { - fprintf(stderr, "server: q: %u %s\n", + } + if (!!ServerUsingModule) { + fprintf(stderr, "server: q: %u %s\n", (unsigned)ServerUsingModule->Bus->GetExecutor()->GetWorkQueueSize(), ServerUsingModule->Session->GetStatusSingleLine().data()); - } + } for (const auto& client : clients) { - fprintf(stderr, "client: q: %u %s\n", + fprintf(stderr, "client: q: %u %s\n", (unsigned)client->Bus->GetExecutor()->GetWorkQueueSize(), client->Session->GetStatusSingleLine().data()); - } - - TStringStream stats; - - bool first = true; - if (!!Server) { - if (!first) { - stats << "\n"; - } - first = false; - stats << "server:\n"; - stats << IndentText(Server->Bus->GetStatus()); - } - if (!!ServerUsingModule) { - if (!first) { - stats << "\n"; - } - first = false; - stats << "server using modules:\n"; - stats << IndentText(ServerUsingModule->Bus->GetStatus()); - } + } + + TStringStream stats; + + bool first = true; + if (!!Server) { + if (!first) { + stats << "\n"; + } + first = false; + stats << "server:\n"; + stats << IndentText(Server->Bus->GetStatus()); + } + if (!!ServerUsingModule) { + if (!first) { + stats << "\n"; + } + first = false; + stats << "server using modules:\n"; + stats << IndentText(ServerUsingModule->Bus->GetStatus()); + } for (const auto& client : clients) { - if (!first) { - stats << "\n"; - } - first = false; - stats << "client:\n"; + if (!first) { + stats << "\n"; + } + first = false; + stats << "client:\n"; stats << IndentText(client->Bus->GetStatus()); - } - + } + TUnbufferedFileOutput("stats").Write(stats.Str()); - } -} - + } +} + int main(int argc, char* argv[]) { - NLWTrace::StartLwtraceFromEnv(); + NLWTrace::StartLwtraceFromEnv(); /* unix foo */ setvbuf(stdout, nullptr, _IONBF, 0); setvbuf(stderr, nullptr, _IONBF, 0); Umask(0); SetAsyncSignalHandler(SIGINT, stopsignal); - SetAsyncSignalHandler(SIGTERM, stopsignal); + SetAsyncSignalHandler(SIGTERM, stopsignal); #ifndef _win_ - SetAsyncSignalHandler(SIGUSR1, stopsignal); + SetAsyncSignalHandler(SIGUSR1, stopsignal); #endif signal(SIGPIPE, SIG_IGN); - NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default(); - opts.AddLongOption('s', "server-port", "server port").RequiredArgument("port").StoreResult(&TheConfig->ServerPort); - opts.AddCharOption('m', "average message size").RequiredArgument("size").StoreResult(&TheConfig->MessageSize); - opts.AddLongOption('c', "server-host", "server hosts").RequiredArgument("host[,host]...").StoreResult(&TheConfig->Nodes); + NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default(); + opts.AddLongOption('s', "server-port", "server port").RequiredArgument("port").StoreResult(&TheConfig->ServerPort); + opts.AddCharOption('m', "average message size").RequiredArgument("size").StoreResult(&TheConfig->MessageSize); + opts.AddLongOption('c', "server-host", "server hosts").RequiredArgument("host[,host]...").StoreResult(&TheConfig->Nodes); opts.AddCharOption('f', "failure rate (rational number between 0 and 1)").RequiredArgument("rate").StoreResult(&TheConfig->Failure); opts.AddCharOption('w', "delay before reply").RequiredArgument("microseconds").StoreResult(&TheConfig->Delay); opts.AddCharOption('r', "run duration").RequiredArgument("seconds").StoreResult(&TheConfig->Run); opts.AddLongOption("client-count", "amount of clients").RequiredArgument("count").StoreResult(&TheConfig->ClientCount).DefaultValue("1"); - opts.AddLongOption("server-use-modules").StoreResult(&TheConfig->ServerUseModules, true); - opts.AddLongOption("on-message-in-pool", "execute OnMessage callback in worker pool") + opts.AddLongOption("server-use-modules").StoreResult(&TheConfig->ServerUseModules, true); + opts.AddLongOption("on-message-in-pool", "execute OnMessage callback in worker pool") .RequiredArgument("BOOL") .StoreResult(&TheConfig->ExecuteOnMessageInWorkerPool); - opts.AddLongOption("on-reply-in-pool", "execute OnReply callback in worker pool") + opts.AddLongOption("on-reply-in-pool", "execute OnReply callback in worker pool") .RequiredArgument("BOOL") .StoreResult(&TheConfig->ExecuteOnReplyInWorkerPool); - opts.AddLongOption("compression", "use compression").RequiredArgument("BOOL").StoreResult(&TheConfig->UseCompression); - opts.AddLongOption("simple-proto").SetFlag(&Config.SimpleProtocol); - opts.AddLongOption("profile").SetFlag(&TheConfig->Profile); - opts.AddLongOption("www-port").RequiredArgument("PORT").StoreResult(&TheConfig->WwwPort); - opts.AddHelpOption(); - - Config.ServerQueueConfig.ConfigureLastGetopt(opts, "server-"); - Config.ServerSessionConfig.ConfigureLastGetopt(opts, "server-"); - Config.ClientQueueConfig.ConfigureLastGetopt(opts, "client-"); - Config.ClientSessionConfig.ConfigureLastGetopt(opts, "client-"); - - opts.SetFreeArgsMax(0); - - NLastGetopt::TOptsParseResult parseResult(&opts, argc, argv); - + opts.AddLongOption("compression", "use compression").RequiredArgument("BOOL").StoreResult(&TheConfig->UseCompression); + opts.AddLongOption("simple-proto").SetFlag(&Config.SimpleProtocol); + opts.AddLongOption("profile").SetFlag(&TheConfig->Profile); + opts.AddLongOption("www-port").RequiredArgument("PORT").StoreResult(&TheConfig->WwwPort); + opts.AddHelpOption(); + + Config.ServerQueueConfig.ConfigureLastGetopt(opts, "server-"); + Config.ServerSessionConfig.ConfigureLastGetopt(opts, "server-"); + Config.ClientQueueConfig.ConfigureLastGetopt(opts, "client-"); + Config.ClientSessionConfig.ConfigureLastGetopt(opts, "client-"); + + opts.SetFreeArgsMax(0); + + NLastGetopt::TOptsParseResult parseResult(&opts, argc, argv); + TheConfig->Print(); - Config.Print(); + Config.Print(); - if (TheConfig->Profile) { - BeginProfiling(); - } - - TIntrusivePtr<TBusWww> www(new TBusWww); - - ServerAddresses = ParseNodes(TheConfig->Nodes); + if (TheConfig->Profile) { + BeginProfiling(); + } + + TIntrusivePtr<TBusWww> www(new TBusWww); + + ServerAddresses = ParseNodes(TheConfig->Nodes); if (TheConfig->ServerPort) { - if (TheConfig->ServerUseModules) { - ServerUsingModule = new TPerftestUsingModule(); - www->RegisterModule(ServerUsingModule.Get()); - } else { - Server = new TPerftestServer(); - www->RegisterServerSession(Server->Session); - } + if (TheConfig->ServerUseModules) { + ServerUsingModule = new TPerftestUsingModule(); + www->RegisterModule(ServerUsingModule.Get()); + } else { + Server = new TPerftestServer(); + www->RegisterServerSession(Server->Session); + } } TVector<TSimpleSharedPtr<NThreading::TLegacyFuture<void, false>>> futures; - - if (ServerAddresses.size() > 0 && TheConfig->ClientCount > 0) { - for (int i = 0; i < TheConfig->ClientCount; ++i) { - TGuard<TMutex> guard(ClientsLock); - Clients.push_back(new TPerftestClient); + + if (ServerAddresses.size() > 0 && TheConfig->ClientCount > 0) { + for (int i = 0; i < TheConfig->ClientCount; ++i) { + TGuard<TMutex> guard(ClientsLock); + Clients.push_back(new TPerftestClient); futures.push_back(new NThreading::TLegacyFuture<void, false>(std::bind(&TPerftestClient::Work, Clients.back()))); - www->RegisterClientSession(Clients.back()->Session); - } + www->RegisterClientSession(Clients.back()->Session); + } } futures.push_back(new NThreading::TLegacyFuture<void, false>(std::bind(&TTestStats::PeriodicallyPrint, std::ref(Stats)))); - - THolder<TBusWwwHttpServer> wwwServer; - if (TheConfig->WwwPort != 0) { - wwwServer.Reset(new TBusWwwHttpServer(www, TheConfig->WwwPort)); - } - - /* sit here until signal terminate our process */ - StopEvent.WaitT(TDuration::Seconds(TheConfig->Run)); - TheExit = true; - StopEvent.Signal(); - - if (!!Server) { - Cerr << "Stopping server\n"; - Server->Stop(); - } - if (!!ServerUsingModule) { - Cerr << "Stopping server (using modules)\n"; - ServerUsingModule->Stop(); - } - + + THolder<TBusWwwHttpServer> wwwServer; + if (TheConfig->WwwPort != 0) { + wwwServer.Reset(new TBusWwwHttpServer(www, TheConfig->WwwPort)); + } + + /* sit here until signal terminate our process */ + StopEvent.WaitT(TDuration::Seconds(TheConfig->Run)); + TheExit = true; + StopEvent.Signal(); + + if (!!Server) { + Cerr << "Stopping server\n"; + Server->Stop(); + } + if (!!ServerUsingModule) { + Cerr << "Stopping server (using modules)\n"; + ServerUsingModule->Stop(); + } + TVector<TSimpleSharedPtr<TPerftestClient>> clients; - { - TGuard<TMutex> guard(ClientsLock); - clients = Clients; - } - - if (!clients.empty()) { - Cerr << "Stopping clients\n"; - + { + TGuard<TMutex> guard(ClientsLock); + clients = Clients; + } + + if (!clients.empty()) { + Cerr << "Stopping clients\n"; + for (auto& client : clients) { client->Stop(); - } - } - - wwwServer.Destroy(); - + } + } + + wwwServer.Destroy(); + for (const auto& future : futures) { future->Get(); - } - - if (TheConfig->Profile) { - EndProfiling(); - } - - Cerr << "***SUCCESS***\n"; + } + + if (TheConfig->Profile) { + EndProfiling(); + } + + Cerr << "***SUCCESS***\n"; return 0; } diff --git a/library/cpp/messagebus/test/perftest/simple_proto.cpp b/library/cpp/messagebus/test/perftest/simple_proto.cpp index 7fab33be6b..19d6c15b9d 100644 --- a/library/cpp/messagebus/test/perftest/simple_proto.cpp +++ b/library/cpp/messagebus/test/perftest/simple_proto.cpp @@ -1,22 +1,22 @@ #include "simple_proto.h" - -#include <util/generic/cast.h> - + +#include <util/generic/cast.h> + #include <typeinfo> - -using namespace NBus; - + +using namespace NBus; + void TSimpleProtocol::Serialize(const TBusMessage* mess, TBuffer& data) { Y_VERIFY(typeid(TSimpleMessage) == typeid(*mess)); - const TSimpleMessage* typed = static_cast<const TSimpleMessage*>(mess); + const TSimpleMessage* typed = static_cast<const TSimpleMessage*>(mess); data.Append((const char*)&typed->Payload, 4); -} - +} + TAutoPtr<TBusMessage> TSimpleProtocol::Deserialize(ui16, TArrayRef<const char> payload) { - if (payload.size() != 4) { + if (payload.size() != 4) { return nullptr; - } - TAutoPtr<TSimpleMessage> r(new TSimpleMessage); - memcpy(&r->Payload, payload.data(), 4); - return r.Release(); -} + } + TAutoPtr<TSimpleMessage> r(new TSimpleMessage); + memcpy(&r->Payload, payload.data(), 4); + return r.Release(); +} diff --git a/library/cpp/messagebus/test/perftest/simple_proto.h b/library/cpp/messagebus/test/perftest/simple_proto.h index 8b0275cf51..4a0cc08db3 100644 --- a/library/cpp/messagebus/test/perftest/simple_proto.h +++ b/library/cpp/messagebus/test/perftest/simple_proto.h @@ -1,29 +1,29 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/ybus.h> - + struct TSimpleMessage: public NBus::TBusMessage { - ui32 Payload; - - TSimpleMessage() + ui32 Payload; + + TSimpleMessage() : TBusMessage(1) , Payload(0) { } - - TSimpleMessage(NBus::ECreateUninitialized) - : TBusMessage(NBus::ECreateUninitialized()) + + TSimpleMessage(NBus::ECreateUninitialized) + : TBusMessage(NBus::ECreateUninitialized()) { } -}; - -struct TSimpleProtocol: public NBus::TBusProtocol { +}; + +struct TSimpleProtocol: public NBus::TBusProtocol { TSimpleProtocol() : NBus::TBusProtocol("simple", 55666) { } - + void Serialize(const NBus::TBusMessage* mess, TBuffer& data) override; - + TAutoPtr<NBus::TBusMessage> Deserialize(ui16 ty, TArrayRef<const char> payload) override; -}; +}; diff --git a/library/cpp/messagebus/test/perftest/stackcollect.diff b/library/cpp/messagebus/test/perftest/stackcollect.diff index a454de3a5d..658f0141b3 100644 --- a/library/cpp/messagebus/test/perftest/stackcollect.diff +++ b/library/cpp/messagebus/test/perftest/stackcollect.diff @@ -1,13 +1,13 @@ -Index: test/perftest/CMakeLists.txt -=================================================================== ---- test/perftest/CMakeLists.txt (revision 1088840) -+++ test/perftest/CMakeLists.txt (working copy) -@@ -3,7 +3,7 @@ PROGRAM(messagebus_perftest) - OWNER(nga) - - PEERDIR( +Index: test/perftest/CMakeLists.txt +=================================================================== +--- test/perftest/CMakeLists.txt (revision 1088840) ++++ test/perftest/CMakeLists.txt (working copy) +@@ -3,7 +3,7 @@ PROGRAM(messagebus_perftest) + OWNER(nga) + + PEERDIR( - library/cpp/execprofile -+ junk/davenger/stackcollect ++ junk/davenger/stackcollect library/cpp/messagebus library/cpp/messagebus/protobuf library/cpp/sighandler diff --git a/library/cpp/messagebus/test/perftest/ya.make b/library/cpp/messagebus/test/perftest/ya.make index 37038ed2a5..24c2848ed5 100644 --- a/library/cpp/messagebus/test/perftest/ya.make +++ b/library/cpp/messagebus/test/perftest/ya.make @@ -1,7 +1,7 @@ -PROGRAM(messagebus_perftest) +PROGRAM(messagebus_perftest) OWNER(g:messagebus) - + PEERDIR( library/cpp/deprecated/threadable library/cpp/execprofile @@ -16,9 +16,9 @@ PEERDIR( ) SRCS( - messages.proto + messages.proto perftest.cpp - simple_proto.cpp + simple_proto.cpp ) END() diff --git a/library/cpp/messagebus/test/ut/count_down_latch.h b/library/cpp/messagebus/test/ut/count_down_latch.h index fb6374e773..5117db5731 100644 --- a/library/cpp/messagebus/test/ut/count_down_latch.h +++ b/library/cpp/messagebus/test/ut/count_down_latch.h @@ -1,30 +1,30 @@ -#pragma once - -#include <util/system/atomic.h> -#include <util/system/event.h> - -class TCountDownLatch { -private: - TAtomic Current; +#pragma once + +#include <util/system/atomic.h> +#include <util/system/event.h> + +class TCountDownLatch { +private: + TAtomic Current; TSystemEvent EventObject; -public: - TCountDownLatch(unsigned initial) - : Current(initial) +public: + TCountDownLatch(unsigned initial) + : Current(initial) { } - - void CountDown() { - if (AtomicDecrement(Current) == 0) { - EventObject.Signal(); - } - } - - void Await() { - EventObject.Wait(); - } - - bool Await(TDuration timeout) { - return EventObject.WaitT(timeout); - } -}; + + void CountDown() { + if (AtomicDecrement(Current) == 0) { + EventObject.Signal(); + } + } + + void Await() { + EventObject.Wait(); + } + + bool Await(TDuration timeout) { + return EventObject.WaitT(timeout); + } +}; diff --git a/library/cpp/messagebus/test/ut/messagebus_ut.cpp b/library/cpp/messagebus/test/ut/messagebus_ut.cpp index 42d4a1e9b2..040f9b7702 100644 --- a/library/cpp/messagebus/test/ut/messagebus_ut.cpp +++ b/library/cpp/messagebus/test/ut/messagebus_ut.cpp @@ -8,104 +8,104 @@ #include <library/cpp/messagebus/misc/test_sync.h> -#include <util/network/sock.h> - +#include <util/network/sock.h> + #include <utility> using namespace NBus; using namespace NBus::NTest; -namespace { - struct TExampleClientSlowOnMessageSent: public TExampleClient { - TAtomic SentCompleted; - +namespace { + struct TExampleClientSlowOnMessageSent: public TExampleClient { + TAtomic SentCompleted; + TSystemEvent ReplyReceived; - - TExampleClientSlowOnMessageSent() - : SentCompleted(0) + + TExampleClientSlowOnMessageSent() + : SentCompleted(0) { } - + ~TExampleClientSlowOnMessageSent() override { - Session->Shutdown(); - } - + Session->Shutdown(); + } + void OnReply(TAutoPtr<TBusMessage> mess, TAutoPtr<TBusMessage> reply) override { Y_VERIFY(AtomicGet(SentCompleted), "must be completed"); - - TExampleClient::OnReply(mess, reply); - - ReplyReceived.Signal(); - } - + + TExampleClient::OnReply(mess, reply); + + ReplyReceived.Signal(); + } + void OnMessageSent(TBusMessage*) override { - Sleep(TDuration::MilliSeconds(100)); - AtomicSet(SentCompleted, 1); - } - }; - -} - + Sleep(TDuration::MilliSeconds(100)); + AtomicSet(SentCompleted, 1); + } + }; + +} + Y_UNIT_TEST_SUITE(TMessageBusTests) { - void TestDestinationTemplate(bool useCompression, bool ackMessageBeforeReply, + void TestDestinationTemplate(bool useCompression, bool ackMessageBeforeReply, const TBusServerSessionConfig& sessionConfig) { - TObjectCountCheck objectCountCheck; - - TExampleServer server; - - TExampleClient client(sessionConfig); - client.CrashOnError = true; - - server.UseCompression = useCompression; - client.UseCompression = useCompression; - - server.AckMessageBeforeSendReply = ackMessageBeforeReply; - - client.SendMessagesWaitReplies(100, server.GetActualListenAddr()); - UNIT_ASSERT_EQUAL(server.Session->GetInFlight(), 0); - UNIT_ASSERT_EQUAL(client.Session->GetInFlight(), 0); - } - + TObjectCountCheck objectCountCheck; + + TExampleServer server; + + TExampleClient client(sessionConfig); + client.CrashOnError = true; + + server.UseCompression = useCompression; + client.UseCompression = useCompression; + + server.AckMessageBeforeSendReply = ackMessageBeforeReply; + + client.SendMessagesWaitReplies(100, server.GetActualListenAddr()); + UNIT_ASSERT_EQUAL(server.Session->GetInFlight(), 0); + UNIT_ASSERT_EQUAL(client.Session->GetInFlight(), 0); + } + Y_UNIT_TEST(TestDestination) { - TestDestinationTemplate(false, false, TBusServerSessionConfig()); - } - + TestDestinationTemplate(false, false, TBusServerSessionConfig()); + } + Y_UNIT_TEST(TestDestinationUsingAck) { - TestDestinationTemplate(false, true, TBusServerSessionConfig()); - } - + TestDestinationTemplate(false, true, TBusServerSessionConfig()); + } + Y_UNIT_TEST(TestDestinationWithCompression) { - TestDestinationTemplate(true, false, TBusServerSessionConfig()); - } - + TestDestinationTemplate(true, false, TBusServerSessionConfig()); + } + Y_UNIT_TEST(TestCork) { - TBusServerSessionConfig config; - config.SendThreshold = 1000000000000; - config.Cork = TDuration::MilliSeconds(10); - TestDestinationTemplate(false, false, config); - // TODO: test for cork hanging - } - + TBusServerSessionConfig config; + config.SendThreshold = 1000000000000; + config.Cork = TDuration::MilliSeconds(10); + TestDestinationTemplate(false, false, config); + // TODO: test for cork hanging + } + Y_UNIT_TEST(TestReconnect) { - if (!IsFixedPortTestAllowed()) { - return; - } - - TObjectCountCheck objectCountCheck; - - unsigned port = FixedPort; - TNetAddr serverAddr("localhost", port); - THolder<TExampleServer> server; - - TBusClientSessionConfig clientConfig; + if (!IsFixedPortTestAllowed()) { + return; + } + + TObjectCountCheck objectCountCheck; + + unsigned port = FixedPort; + TNetAddr serverAddr("localhost", port); + THolder<TExampleServer> server; + + TBusClientSessionConfig clientConfig; clientConfig.RetryInterval = 0; - TExampleClient client(clientConfig); - - server.Reset(new TExampleServer(port, "TExampleServer 1")); - - client.SendMessagesWaitReplies(17, serverAddr); - - server.Destroy(); + TExampleClient client(clientConfig); + + server.Reset(new TExampleServer(port, "TExampleServer 1")); + + client.SendMessagesWaitReplies(17, serverAddr); + + server.Destroy(); // Making the client to detect disconnection. client.SendMessages(1, serverAddr); @@ -116,11 +116,11 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { } UNIT_ASSERT_VALUES_EQUAL(MESSAGE_CONNECT_FAILED, error); - server.Reset(new TExampleServer(port, "TExampleServer 2")); - - client.SendMessagesWaitReplies(19, serverAddr); - } - + server.Reset(new TExampleServer(port, "TExampleServer 2")); + + client.SendMessagesWaitReplies(19, serverAddr); + } + struct TestNoServerImplClient: public TExampleClient { TTestSync TestSync; int failures = 0; @@ -145,8 +145,8 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { }; void TestNoServerImpl(unsigned port, bool oneWay) { - TNetAddr noServerAddr("localhost", port); - + TNetAddr noServerAddr("localhost", port); + TestNoServerImplClient client; int count = 0; @@ -174,167 +174,167 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { void HangingServerImpl(unsigned port) { TNetAddr noServerAddr("localhost", port); - TExampleClient client; - - int count = 0; - for (;; ++count) { - TAutoPtr<TBusMessage> message(new TExampleRequest(&client.Proto.RequestCount)); - EMessageStatus status = client.Session->SendMessageAutoPtr(message, &noServerAddr); - if (status == MESSAGE_BUSY) { - break; - } - UNIT_ASSERT_VALUES_EQUAL(int(MESSAGE_OK), int(status)); - - if (count == 0) { - // lame way to wait until it is connected - Sleep(TDuration::MilliSeconds(10)); - } - } - - UNIT_ASSERT_VALUES_EQUAL(client.Session->GetConfig()->MaxInFlight, count); - } - + TExampleClient client; + + int count = 0; + for (;; ++count) { + TAutoPtr<TBusMessage> message(new TExampleRequest(&client.Proto.RequestCount)); + EMessageStatus status = client.Session->SendMessageAutoPtr(message, &noServerAddr); + if (status == MESSAGE_BUSY) { + break; + } + UNIT_ASSERT_VALUES_EQUAL(int(MESSAGE_OK), int(status)); + + if (count == 0) { + // lame way to wait until it is connected + Sleep(TDuration::MilliSeconds(10)); + } + } + + UNIT_ASSERT_VALUES_EQUAL(client.Session->GetConfig()->MaxInFlight, count); + } + Y_UNIT_TEST(TestHangindServer) { - TObjectCountCheck objectCountCheck; - - THangingServer server(0); - + TObjectCountCheck objectCountCheck; + + THangingServer server(0); + HangingServerImpl(server.GetPort()); - } - + } + Y_UNIT_TEST(TestNoServer) { - TObjectCountCheck objectCountCheck; - + TObjectCountCheck objectCountCheck; + TestNoServerImpl(17, false); - } - + } + Y_UNIT_TEST(PauseInput) { - TObjectCountCheck objectCountCheck; - - TExampleServer server; - server.Session->PauseInput(true); - - TBusClientSessionConfig clientConfig; - clientConfig.MaxInFlight = 1000; - TExampleClient client(clientConfig); - - client.SendMessages(100, server.GetActualListenAddr()); - - server.TestSync.Check(0); - - server.Session->PauseInput(false); - - server.TestSync.WaitFor(100); - - client.WaitReplies(); - - server.Session->PauseInput(true); - - client.SendMessages(200, server.GetActualListenAddr()); - - server.TestSync.Check(100); - - server.Session->PauseInput(false); - - server.TestSync.WaitFor(300); - - client.WaitReplies(); - } - + TObjectCountCheck objectCountCheck; + + TExampleServer server; + server.Session->PauseInput(true); + + TBusClientSessionConfig clientConfig; + clientConfig.MaxInFlight = 1000; + TExampleClient client(clientConfig); + + client.SendMessages(100, server.GetActualListenAddr()); + + server.TestSync.Check(0); + + server.Session->PauseInput(false); + + server.TestSync.WaitFor(100); + + client.WaitReplies(); + + server.Session->PauseInput(true); + + client.SendMessages(200, server.GetActualListenAddr()); + + server.TestSync.Check(100); + + server.Session->PauseInput(false); + + server.TestSync.WaitFor(300); + + client.WaitReplies(); + } + struct TSendTimeoutCheckerExampleClient: public TExampleClient { - static TBusClientSessionConfig SessionConfig(bool periodLessThanConnectTimeout) { - TBusClientSessionConfig sessionConfig; - if (periodLessThanConnectTimeout) { - sessionConfig.SendTimeout = 1; - sessionConfig.Secret.TimeoutPeriod = TDuration::MilliSeconds(50); - } else { - sessionConfig.SendTimeout = 50; - sessionConfig.Secret.TimeoutPeriod = TDuration::MilliSeconds(1); - } - return sessionConfig; - } - - TSendTimeoutCheckerExampleClient(bool periodLessThanConnectTimeout) - : TExampleClient(SessionConfig(periodLessThanConnectTimeout)) + static TBusClientSessionConfig SessionConfig(bool periodLessThanConnectTimeout) { + TBusClientSessionConfig sessionConfig; + if (periodLessThanConnectTimeout) { + sessionConfig.SendTimeout = 1; + sessionConfig.Secret.TimeoutPeriod = TDuration::MilliSeconds(50); + } else { + sessionConfig.SendTimeout = 50; + sessionConfig.Secret.TimeoutPeriod = TDuration::MilliSeconds(1); + } + return sessionConfig; + } + + TSendTimeoutCheckerExampleClient(bool periodLessThanConnectTimeout) + : TExampleClient(SessionConfig(periodLessThanConnectTimeout)) { } - + ~TSendTimeoutCheckerExampleClient() override { - Session->Shutdown(); - } - + Session->Shutdown(); + } + TSystemEvent ErrorHappened; - + void OnError(TAutoPtr<TBusMessage>, EMessageStatus status) override { Y_VERIFY(status == MESSAGE_CONNECT_FAILED || status == MESSAGE_TIMEOUT, "got status: %s", ToString(status).data()); - ErrorHappened.Signal(); - } - }; - - void NoServer_SendTimeout_Callback_Impl(bool periodLessThanConnectTimeout) { - TObjectCountCheck objectCountCheck; - - TNetAddr serverAddr("localhost", 17); - - TSendTimeoutCheckerExampleClient client(periodLessThanConnectTimeout); - - client.SendMessages(1, serverAddr); - - client.ErrorHappened.WaitI(); - } - + ErrorHappened.Signal(); + } + }; + + void NoServer_SendTimeout_Callback_Impl(bool periodLessThanConnectTimeout) { + TObjectCountCheck objectCountCheck; + + TNetAddr serverAddr("localhost", 17); + + TSendTimeoutCheckerExampleClient client(periodLessThanConnectTimeout); + + client.SendMessages(1, serverAddr); + + client.ErrorHappened.WaitI(); + } + Y_UNIT_TEST(NoServer_SendTimeout_Callback_PeriodLess) { - NoServer_SendTimeout_Callback_Impl(true); - } - + NoServer_SendTimeout_Callback_Impl(true); + } + Y_UNIT_TEST(NoServer_SendTimeout_Callback_TimeoutLess) { - NoServer_SendTimeout_Callback_Impl(false); - } - + NoServer_SendTimeout_Callback_Impl(false); + } + Y_UNIT_TEST(TestOnReplyCalledAfterOnMessageSent) { - TObjectCountCheck objectCountCheck; - - TExampleServer server; - TNetAddr serverAddr = server.GetActualListenAddr(); - TExampleClientSlowOnMessageSent client; - - TAutoPtr<TExampleRequest> message(new TExampleRequest(&client.Proto.RequestCount)); - EMessageStatus s = client.Session->SendMessageAutoPtr(message, &serverAddr); - UNIT_ASSERT_EQUAL(s, MESSAGE_OK); - - UNIT_ASSERT(client.ReplyReceived.WaitT(TDuration::Seconds(5))); - } - - struct TDelayReplyServer: public TBusServerHandlerError { - TBusMessageQueuePtr Bus; - TExampleProtocol Proto; + TObjectCountCheck objectCountCheck; + + TExampleServer server; + TNetAddr serverAddr = server.GetActualListenAddr(); + TExampleClientSlowOnMessageSent client; + + TAutoPtr<TExampleRequest> message(new TExampleRequest(&client.Proto.RequestCount)); + EMessageStatus s = client.Session->SendMessageAutoPtr(message, &serverAddr); + UNIT_ASSERT_EQUAL(s, MESSAGE_OK); + + UNIT_ASSERT(client.ReplyReceived.WaitT(TDuration::Seconds(5))); + } + + struct TDelayReplyServer: public TBusServerHandlerError { + TBusMessageQueuePtr Bus; + TExampleProtocol Proto; TSystemEvent MessageReceivedEvent; // 1 wait for 1 message - TBusServerSessionPtr Session; + TBusServerSessionPtr Session; TMutex Lock_; TDeque<TAutoPtr<TOnMessageContext>> DelayedMessages; - + TDelayReplyServer() : MessageReceivedEvent(TEventResetType::rAuto) { - Bus = CreateMessageQueue("TDelayReplyServer"); - TBusServerSessionConfig sessionConfig; - sessionConfig.SendTimeout = 1000; - sessionConfig.TotalTimeout = 2001; - Session = TBusServerSession::Create(&Proto, this, sessionConfig, Bus); - if (!Session) { - ythrow yexception() << "Failed to create destination session"; - } - } - + Bus = CreateMessageQueue("TDelayReplyServer"); + TBusServerSessionConfig sessionConfig; + sessionConfig.SendTimeout = 1000; + sessionConfig.TotalTimeout = 2001; + Session = TBusServerSession::Create(&Proto, this, sessionConfig, Bus); + if (!Session) { + ythrow yexception() << "Failed to create destination session"; + } + } + void OnMessage(TOnMessageContext& mess) override { Y_VERIFY(mess.IsConnectionAlive(), "connection should be alive here"); TAutoPtr<TOnMessageContext> delayedMsg(new TOnMessageContext); delayedMsg->Swap(mess); auto g(Guard(Lock_)); DelayedMessages.push_back(delayedMsg); - MessageReceivedEvent.Signal(); + MessageReceivedEvent.Signal(); } - + bool CheckClientIsAlive() { auto g(Guard(Lock_)); for (auto& delayedMessage : DelayedMessages) { @@ -370,252 +370,252 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { msg.SendReplyMove(reply); } } - + size_t GetDelayedMessageCount() const { auto g(Guard(Lock_)); return DelayedMessages.size(); - } - + } + void OnError(TAutoPtr<TBusMessage> mess, EMessageStatus status) override { Y_UNUSED(mess); Y_VERIFY(status == MESSAGE_SHUTDOWN, "only shutdown allowed, got %s", ToString(status).data()); - } - }; - + } + }; + Y_UNIT_TEST(TestReplyCalledAfterClientDisconnected) { - TObjectCountCheck objectCountCheck; - - TDelayReplyServer server; - - THolder<TExampleClient> client(new TExampleClient); - - client->SendMessages(1, TNetAddr("localhost", server.Session->GetActualListenPort())); - - UNIT_ASSERT(server.MessageReceivedEvent.WaitT(TDuration::Seconds(5))); - - UNIT_ASSERT_VALUES_EQUAL(1, server.Session->GetInFlight()); - - client.Destroy(); - + TObjectCountCheck objectCountCheck; + + TDelayReplyServer server; + + THolder<TExampleClient> client(new TExampleClient); + + client->SendMessages(1, TNetAddr("localhost", server.Session->GetActualListenPort())); + + UNIT_ASSERT(server.MessageReceivedEvent.WaitT(TDuration::Seconds(5))); + + UNIT_ASSERT_VALUES_EQUAL(1, server.Session->GetInFlight()); + + client.Destroy(); + UNIT_WAIT_FOR(server.CheckClientIsDead()); - + server.ReplyToDelayedMessages(); - // wait until all server message are delivered - UNIT_WAIT_FOR(0 == server.Session->GetInFlight()); - } - - struct TPackUnpackServer: public TBusServerHandlerError { - TBusMessageQueuePtr Bus; - TExampleProtocol Proto; + // wait until all server message are delivered + UNIT_WAIT_FOR(0 == server.Session->GetInFlight()); + } + + struct TPackUnpackServer: public TBusServerHandlerError { + TBusMessageQueuePtr Bus; + TExampleProtocol Proto; TSystemEvent MessageReceivedEvent; TSystemEvent ClientDiedEvent; - TBusServerSessionPtr Session; - - TPackUnpackServer() { - Bus = CreateMessageQueue("TPackUnpackServer"); - TBusServerSessionConfig sessionConfig; - Session = TBusServerSession::Create(&Proto, this, sessionConfig, Bus); - } - + TBusServerSessionPtr Session; + + TPackUnpackServer() { + Bus = CreateMessageQueue("TPackUnpackServer"); + TBusServerSessionConfig sessionConfig; + Session = TBusServerSession::Create(&Proto, this, sessionConfig, Bus); + } + void OnMessage(TOnMessageContext& mess) override { - TBusIdentity ident; - mess.AckMessage(ident); - - char packed[BUS_IDENTITY_PACKED_SIZE]; - ident.Pack(packed); - TBusIdentity resurrected; - resurrected.Unpack(packed); - - mess.GetSession()->SendReply(resurrected, new TExampleResponse(&Proto.ResponseCount)); - } - + TBusIdentity ident; + mess.AckMessage(ident); + + char packed[BUS_IDENTITY_PACKED_SIZE]; + ident.Pack(packed); + TBusIdentity resurrected; + resurrected.Unpack(packed); + + mess.GetSession()->SendReply(resurrected, new TExampleResponse(&Proto.ResponseCount)); + } + void OnError(TAutoPtr<TBusMessage> mess, EMessageStatus status) override { Y_UNUSED(mess); Y_VERIFY(status == MESSAGE_SHUTDOWN, "only shutdown allowed"); - } - }; - + } + }; + Y_UNIT_TEST(PackUnpack) { - TObjectCountCheck objectCountCheck; - - TPackUnpackServer server; - - THolder<TExampleClient> client(new TExampleClient); - - client->SendMessagesWaitReplies(1, TNetAddr("localhost", server.Session->GetActualListenPort())); - } - + TObjectCountCheck objectCountCheck; + + TPackUnpackServer server; + + THolder<TExampleClient> client(new TExampleClient); + + client->SendMessagesWaitReplies(1, TNetAddr("localhost", server.Session->GetActualListenPort())); + } + Y_UNIT_TEST(ClientRequestTooLarge) { - TObjectCountCheck objectCountCheck; - - TExampleServer server; - - TBusClientSessionConfig clientConfig; - clientConfig.MaxMessageSize = 100; - TExampleClient client(clientConfig); - - client.DataSize = 10; - client.SendMessagesWaitReplies(1, server.GetActualListenAddr()); - - client.DataSize = 1000; - client.SendMessages(1, server.GetActualListenAddr()); - client.WaitForError(MESSAGE_MESSAGE_TOO_LARGE); - - client.DataSize = 20; - client.SendMessagesWaitReplies(10, server.GetActualListenAddr()); - - client.DataSize = 10000; - client.SendMessages(1, server.GetActualListenAddr()); - client.WaitForError(MESSAGE_MESSAGE_TOO_LARGE); - } - + TObjectCountCheck objectCountCheck; + + TExampleServer server; + + TBusClientSessionConfig clientConfig; + clientConfig.MaxMessageSize = 100; + TExampleClient client(clientConfig); + + client.DataSize = 10; + client.SendMessagesWaitReplies(1, server.GetActualListenAddr()); + + client.DataSize = 1000; + client.SendMessages(1, server.GetActualListenAddr()); + client.WaitForError(MESSAGE_MESSAGE_TOO_LARGE); + + client.DataSize = 20; + client.SendMessagesWaitReplies(10, server.GetActualListenAddr()); + + client.DataSize = 10000; + client.SendMessages(1, server.GetActualListenAddr()); + client.WaitForError(MESSAGE_MESSAGE_TOO_LARGE); + } + struct TServerForResponseTooLarge: public TExampleServer { - TTestSync TestSync; - - static TBusServerSessionConfig Config() { - TBusServerSessionConfig config; - config.MaxMessageSize = 100; - return config; - } - - TServerForResponseTooLarge() - : TExampleServer("TServerForResponseTooLarge", Config()) + TTestSync TestSync; + + static TBusServerSessionConfig Config() { + TBusServerSessionConfig config; + config.MaxMessageSize = 100; + return config; + } + + TServerForResponseTooLarge() + : TExampleServer("TServerForResponseTooLarge", Config()) { } - + ~TServerForResponseTooLarge() override { - Session->Shutdown(); - } - + Session->Shutdown(); + } + void OnMessage(TOnMessageContext& mess) override { - TAutoPtr<TBusMessage> response; - - if (TestSync.Get() == 0) { - TestSync.CheckAndIncrement(0); - response.Reset(new TExampleResponse(&Proto.ResponseCount, 1000)); - } else { - TestSync.WaitForAndIncrement(3); - response.Reset(new TExampleResponse(&Proto.ResponseCount, 10)); - } - - mess.SendReplyMove(response); - } - + TAutoPtr<TBusMessage> response; + + if (TestSync.Get() == 0) { + TestSync.CheckAndIncrement(0); + response.Reset(new TExampleResponse(&Proto.ResponseCount, 1000)); + } else { + TestSync.WaitForAndIncrement(3); + response.Reset(new TExampleResponse(&Proto.ResponseCount, 10)); + } + + mess.SendReplyMove(response); + } + void OnError(TAutoPtr<TBusMessage>, EMessageStatus status) override { - TestSync.WaitForAndIncrement(1); - + TestSync.WaitForAndIncrement(1); + Y_VERIFY(status == MESSAGE_MESSAGE_TOO_LARGE, "status"); - } - }; - + } + }; + Y_UNIT_TEST(ServerResponseTooLarge) { - TObjectCountCheck objectCountCheck; - - TServerForResponseTooLarge server; - - TExampleClient client; - client.DataSize = 10; - - client.SendMessages(1, server.GetActualListenAddr()); - server.TestSync.WaitForAndIncrement(2); - client.ResetCounters(); - - client.SendMessages(1, server.GetActualListenAddr()); - - client.WorkDone.WaitI(); - - server.TestSync.CheckAndIncrement(4); - - UNIT_ASSERT_VALUES_EQUAL(1, client.Session->GetInFlight()); - } - + TObjectCountCheck objectCountCheck; + + TServerForResponseTooLarge server; + + TExampleClient client; + client.DataSize = 10; + + client.SendMessages(1, server.GetActualListenAddr()); + server.TestSync.WaitForAndIncrement(2); + client.ResetCounters(); + + client.SendMessages(1, server.GetActualListenAddr()); + + client.WorkDone.WaitI(); + + server.TestSync.CheckAndIncrement(4); + + UNIT_ASSERT_VALUES_EQUAL(1, client.Session->GetInFlight()); + } + struct TServerForRequestTooLarge: public TExampleServer { - TTestSync TestSync; - - static TBusServerSessionConfig Config() { - TBusServerSessionConfig config; - config.MaxMessageSize = 100; - return config; - } - - TServerForRequestTooLarge() - : TExampleServer("TServerForRequestTooLarge", Config()) + TTestSync TestSync; + + static TBusServerSessionConfig Config() { + TBusServerSessionConfig config; + config.MaxMessageSize = 100; + return config; + } + + TServerForRequestTooLarge() + : TExampleServer("TServerForRequestTooLarge", Config()) { } - + ~TServerForRequestTooLarge() override { - Session->Shutdown(); - } - + Session->Shutdown(); + } + void OnMessage(TOnMessageContext& req) override { - unsigned n = TestSync.Get(); - if (n < 2) { - TestSync.CheckAndIncrement(n); - TAutoPtr<TExampleResponse> resp(new TExampleResponse(&Proto.ResponseCount, 10)); - req.SendReplyMove(resp); - } else { + unsigned n = TestSync.Get(); + if (n < 2) { + TestSync.CheckAndIncrement(n); + TAutoPtr<TExampleResponse> resp(new TExampleResponse(&Proto.ResponseCount, 10)); + req.SendReplyMove(resp); + } else { Y_FAIL("wrong"); - } - } - }; - + } + } + }; + Y_UNIT_TEST(ServerRequestTooLarge) { - TObjectCountCheck objectCountCheck; - - TServerForRequestTooLarge server; - - TExampleClient client; - client.DataSize = 10; - - client.SendMessagesWaitReplies(2, server.GetActualListenAddr()); - - server.TestSync.CheckAndIncrement(2); - - client.DataSize = 200; - client.SendMessages(1, server.GetActualListenAddr()); - // server closes connection, so MESSAGE_DELIVERY_FAILED is returned to client - client.WaitForError(MESSAGE_DELIVERY_FAILED); - } - + TObjectCountCheck objectCountCheck; + + TServerForRequestTooLarge server; + + TExampleClient client; + client.DataSize = 10; + + client.SendMessagesWaitReplies(2, server.GetActualListenAddr()); + + server.TestSync.CheckAndIncrement(2); + + client.DataSize = 200; + client.SendMessages(1, server.GetActualListenAddr()); + // server closes connection, so MESSAGE_DELIVERY_FAILED is returned to client + client.WaitForError(MESSAGE_DELIVERY_FAILED); + } + Y_UNIT_TEST(ClientResponseTooLarge) { - TObjectCountCheck objectCountCheck; - - TExampleServer server; - - server.DataSize = 10; - - TBusClientSessionConfig clientSessionConfig; - clientSessionConfig.MaxMessageSize = 100; - TExampleClient client(clientSessionConfig); - client.DataSize = 10; - - client.SendMessagesWaitReplies(3, server.GetActualListenAddr()); - - server.DataSize = 1000; - - client.SendMessages(1, server.GetActualListenAddr()); - client.WaitForError(MESSAGE_DELIVERY_FAILED); - } - + TObjectCountCheck objectCountCheck; + + TExampleServer server; + + server.DataSize = 10; + + TBusClientSessionConfig clientSessionConfig; + clientSessionConfig.MaxMessageSize = 100; + TExampleClient client(clientSessionConfig); + client.DataSize = 10; + + client.SendMessagesWaitReplies(3, server.GetActualListenAddr()); + + server.DataSize = 1000; + + client.SendMessages(1, server.GetActualListenAddr()); + client.WaitForError(MESSAGE_DELIVERY_FAILED); + } + Y_UNIT_TEST(ServerUnknownMessage) { - TObjectCountCheck objectCountCheck; - - TExampleServer server; - TNetAddr serverAddr = server.GetActualListenAddr(); - - TExampleClient client; - - client.SendMessagesWaitReplies(2, serverAddr); - - TAutoPtr<TBusMessage> req(new TExampleRequest(&client.Proto.RequestCount)); - req->GetHeader()->Type = 11; - client.Session->SendMessageAutoPtr(req, &serverAddr); - client.MessageCount = 1; - - client.WaitForError(MESSAGE_DELIVERY_FAILED); - } - + TObjectCountCheck objectCountCheck; + + TExampleServer server; + TNetAddr serverAddr = server.GetActualListenAddr(); + + TExampleClient client; + + client.SendMessagesWaitReplies(2, serverAddr); + + TAutoPtr<TBusMessage> req(new TExampleRequest(&client.Proto.RequestCount)); + req->GetHeader()->Type = 11; + client.Session->SendMessageAutoPtr(req, &serverAddr); + client.MessageCount = 1; + + client.WaitForError(MESSAGE_DELIVERY_FAILED); + } + Y_UNIT_TEST(ServerMessageReservedIds) { TObjectCountCheck objectCountCheck; @@ -642,18 +642,18 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { } Y_UNIT_TEST(TestGetInFlightForDestination) { - TObjectCountCheck objectCountCheck; - - TDelayReplyServer server; - - TExampleClient client; - - TNetAddr addr("localhost", server.Session->GetActualListenPort()); - - UNIT_ASSERT_VALUES_EQUAL(size_t(0), client.Session->GetInFlight(addr)); - - client.SendMessages(2, &addr); - + TObjectCountCheck objectCountCheck; + + TDelayReplyServer server; + + TExampleClient client; + + TNetAddr addr("localhost", server.Session->GetActualListenPort()); + + UNIT_ASSERT_VALUES_EQUAL(size_t(0), client.Session->GetInFlight(addr)); + + client.SendMessages(2, &addr); + for (size_t i = 0; i < 5; ++i) { // One MessageReceivedEvent indicates one message, we need to wait for two UNIT_ASSERT(server.MessageReceivedEvent.WaitT(TDuration::Seconds(5))); @@ -662,98 +662,98 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { } } UNIT_ASSERT_VALUES_EQUAL(server.GetDelayedMessageCount(), 2); - - size_t inFlight = client.Session->GetInFlight(addr); - // 4 is for messagebus1 that adds inFlight counter twice for some reason - UNIT_ASSERT(inFlight == 2 || inFlight == 4); - + + size_t inFlight = client.Session->GetInFlight(addr); + // 4 is for messagebus1 that adds inFlight counter twice for some reason + UNIT_ASSERT(inFlight == 2 || inFlight == 4); + UNIT_ASSERT(server.CheckClientIsAlive()); - + server.ReplyToDelayedMessages(); - client.WaitReplies(); - } - + client.WaitReplies(); + } + struct TResetAfterSendOneWayErrorInCallbackClient: public TExampleClient { - TTestSync TestSync; - - static TBusClientSessionConfig SessionConfig() { - TBusClientSessionConfig config; - // 1 ms is not enough when test is running under valgrind - config.ConnectTimeout = 10; - config.SendTimeout = 10; - config.Secret.TimeoutPeriod = TDuration::MilliSeconds(1); - return config; - } - - TResetAfterSendOneWayErrorInCallbackClient() - : TExampleClient(SessionConfig()) - { - } - + TTestSync TestSync; + + static TBusClientSessionConfig SessionConfig() { + TBusClientSessionConfig config; + // 1 ms is not enough when test is running under valgrind + config.ConnectTimeout = 10; + config.SendTimeout = 10; + config.Secret.TimeoutPeriod = TDuration::MilliSeconds(1); + return config; + } + + TResetAfterSendOneWayErrorInCallbackClient() + : TExampleClient(SessionConfig()) + { + } + ~TResetAfterSendOneWayErrorInCallbackClient() override { - Session->Shutdown(); - } - + Session->Shutdown(); + } + void OnError(TAutoPtr<TBusMessage> mess, EMessageStatus status) override { - TestSync.WaitForAndIncrement(0); + TestSync.WaitForAndIncrement(0); Y_VERIFY(status == MESSAGE_CONNECT_FAILED || status == MESSAGE_TIMEOUT, "must be connection failed, got %s", ToString(status).data()); - mess.Destroy(); - TestSync.CheckAndIncrement(1); - } - }; - + mess.Destroy(); + TestSync.CheckAndIncrement(1); + } + }; + Y_UNIT_TEST(ResetAfterSendOneWayErrorInCallback) { - TObjectCountCheck objectCountCheck; - - TNetAddr noServerAddr("localhost", 17); - - TResetAfterSendOneWayErrorInCallbackClient client; - - EMessageStatus ok = client.Session->SendMessageOneWayMove(new TExampleRequest(&client.Proto.RequestCount), &noServerAddr); - UNIT_ASSERT_VALUES_EQUAL(MESSAGE_OK, ok); - - client.TestSync.WaitForAndIncrement(2); - } - + TObjectCountCheck objectCountCheck; + + TNetAddr noServerAddr("localhost", 17); + + TResetAfterSendOneWayErrorInCallbackClient client; + + EMessageStatus ok = client.Session->SendMessageOneWayMove(new TExampleRequest(&client.Proto.RequestCount), &noServerAddr); + UNIT_ASSERT_VALUES_EQUAL(MESSAGE_OK, ok); + + client.TestSync.WaitForAndIncrement(2); + } + struct TResetAfterSendMessageOneWayDuringShutdown: public TExampleClient { - TTestSync TestSync; - + TTestSync TestSync; + ~TResetAfterSendMessageOneWayDuringShutdown() override { - Session->Shutdown(); - } - + Session->Shutdown(); + } + void OnError(TAutoPtr<TBusMessage> message, EMessageStatus status) override { - TestSync.CheckAndIncrement(0); - + TestSync.CheckAndIncrement(0); + Y_VERIFY(status == MESSAGE_CONNECT_FAILED, "must be MESSAGE_CONNECT_FAILED, got %s", ToString(status).data()); - - // check reset is possible here - message->Reset(); - + + // check reset is possible here + message->Reset(); + // intentionally don't destroy the message // we will try to resend it Y_UNUSED(message.Release()); - TestSync.CheckAndIncrement(1); - } - }; - + TestSync.CheckAndIncrement(1); + } + }; + Y_UNIT_TEST(ResetAfterSendMessageOneWayDuringShutdown) { - TObjectCountCheck objectCountCheck; - - TNetAddr noServerAddr("localhost", 17); - - TResetAfterSendMessageOneWayDuringShutdown client; - + TObjectCountCheck objectCountCheck; + + TNetAddr noServerAddr("localhost", 17); + + TResetAfterSendMessageOneWayDuringShutdown client; + TExampleRequest* message = new TExampleRequest(&client.Proto.RequestCount); EMessageStatus ok = client.Session->SendMessageOneWay(message, &noServerAddr); - UNIT_ASSERT_VALUES_EQUAL(MESSAGE_OK, ok); - + UNIT_ASSERT_VALUES_EQUAL(MESSAGE_OK, ok); + client.TestSync.WaitForAndIncrement(2); - client.Session->Shutdown(); - + client.Session->Shutdown(); + ok = client.Session->SendMessageOneWay(message); Y_VERIFY(ok == MESSAGE_SHUTDOWN, "must be shutdown when sending during shutdown, got %s", ToString(ok).data()); @@ -762,148 +762,148 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { client.TestSync.CheckAndIncrement(3); delete message; - } - + } + Y_UNIT_TEST(ResetAfterSendOneWayErrorInReturn) { - TObjectCountCheck objectCountCheck; - + TObjectCountCheck objectCountCheck; + TestNoServerImpl(17, true); - } - + } + struct TResetAfterSendOneWaySuccessClient: public TExampleClient { - TTestSync TestSync; - + TTestSync TestSync; + ~TResetAfterSendOneWaySuccessClient() override { - Session->Shutdown(); - } - + Session->Shutdown(); + } + void OnMessageSentOneWay(TAutoPtr<TBusMessage> sent) override { - TestSync.WaitForAndIncrement(0); - sent->Reset(); - TestSync.CheckAndIncrement(1); - } - }; - + TestSync.WaitForAndIncrement(0); + sent->Reset(); + TestSync.CheckAndIncrement(1); + } + }; + Y_UNIT_TEST(ResetAfterSendOneWaySuccess) { - TObjectCountCheck objectCountCheck; - - TExampleServer server; - TNetAddr serverAddr = server.GetActualListenAddr(); - - TResetAfterSendOneWaySuccessClient client; - - EMessageStatus ok = client.Session->SendMessageOneWay(new TExampleRequest(&client.Proto.RequestCount), &serverAddr); - UNIT_ASSERT_VALUES_EQUAL(MESSAGE_OK, ok); - // otherwize message might go to OnError(MESSAGE_SHUTDOWN) - server.WaitForOnMessageCount(1); - - client.TestSync.WaitForAndIncrement(2); - } - + TObjectCountCheck objectCountCheck; + + TExampleServer server; + TNetAddr serverAddr = server.GetActualListenAddr(); + + TResetAfterSendOneWaySuccessClient client; + + EMessageStatus ok = client.Session->SendMessageOneWay(new TExampleRequest(&client.Proto.RequestCount), &serverAddr); + UNIT_ASSERT_VALUES_EQUAL(MESSAGE_OK, ok); + // otherwize message might go to OnError(MESSAGE_SHUTDOWN) + server.WaitForOnMessageCount(1); + + client.TestSync.WaitForAndIncrement(2); + } + Y_UNIT_TEST(GetStatus) { - TObjectCountCheck objectCountCheck; - - TExampleServer server; - - TExampleClient client; - // make sure connected - client.SendMessagesWaitReplies(3, server.GetActualListenAddr()); - - server.Bus->GetStatus(); - server.Bus->GetStatus(); - server.Bus->GetStatus(); - - client.Bus->GetStatus(); - client.Bus->GetStatus(); - client.Bus->GetStatus(); - } - + TObjectCountCheck objectCountCheck; + + TExampleServer server; + + TExampleClient client; + // make sure connected + client.SendMessagesWaitReplies(3, server.GetActualListenAddr()); + + server.Bus->GetStatus(); + server.Bus->GetStatus(); + server.Bus->GetStatus(); + + client.Bus->GetStatus(); + client.Bus->GetStatus(); + client.Bus->GetStatus(); + } + Y_UNIT_TEST(BindOnRandomPort) { - TObjectCountCheck objectCountCheck; - - TBusServerSessionConfig serverConfig; - TExampleServer server; - - TExampleClient client; - TNetAddr addr(TNetAddr("127.0.0.1", server.Session->GetActualListenPort())); - client.SendMessagesWaitReplies(3, &addr); - } - + TObjectCountCheck objectCountCheck; + + TBusServerSessionConfig serverConfig; + TExampleServer server; + + TExampleClient client; + TNetAddr addr(TNetAddr("127.0.0.1", server.Session->GetActualListenPort())); + client.SendMessagesWaitReplies(3, &addr); + } + Y_UNIT_TEST(UnbindOnShutdown) { - TBusMessageQueuePtr queue(CreateMessageQueue()); - - TExampleProtocol proto; - TBusServerHandlerError handler; - TBusServerSessionPtr session = TBusServerSession::Create( + TBusMessageQueuePtr queue(CreateMessageQueue()); + + TExampleProtocol proto; + TBusServerHandlerError handler; + TBusServerSessionPtr session = TBusServerSession::Create( &proto, &handler, TBusServerSessionConfig(), queue); - - unsigned port = session->GetActualListenPort(); - UNIT_ASSERT(port > 0); - - session->Shutdown(); - - // fails is Shutdown() didn't unbind - THangingServer hangingServer(port); - } - + + unsigned port = session->GetActualListenPort(); + UNIT_ASSERT(port > 0); + + session->Shutdown(); + + // fails is Shutdown() didn't unbind + THangingServer hangingServer(port); + } + Y_UNIT_TEST(VersionNegotiation) { - TObjectCountCheck objectCountCheck; - - TExampleServer server; - - TSockAddrInet addr(IpFromString("127.0.0.1"), server.Session->GetActualListenPort()); - - TInetStreamSocket socket; - int r1 = socket.Connect(&addr); - UNIT_ASSERT(r1 >= 0); - - TStreamSocketOutput output(&socket); - - TBusHeader request; - Zero(request); - request.Size = sizeof(request); - request.SetVersionInternal(0xF); // max - output.Write(&request, sizeof(request)); - + TObjectCountCheck objectCountCheck; + + TExampleServer server; + + TSockAddrInet addr(IpFromString("127.0.0.1"), server.Session->GetActualListenPort()); + + TInetStreamSocket socket; + int r1 = socket.Connect(&addr); + UNIT_ASSERT(r1 >= 0); + + TStreamSocketOutput output(&socket); + + TBusHeader request; + Zero(request); + request.Size = sizeof(request); + request.SetVersionInternal(0xF); // max + output.Write(&request, sizeof(request)); + UNIT_ASSERT_VALUES_EQUAL(IsVersionNegotiation(request), true); - TStreamSocketInput input(&socket); - - TBusHeader response; - size_t pos = 0; - - while (pos < sizeof(response)) { + TStreamSocketInput input(&socket); + + TBusHeader response; + size_t pos = 0; + + while (pos < sizeof(response)) { size_t count = input.Read(((char*)&response) + pos, sizeof(response) - pos); - pos += count; - } - - UNIT_ASSERT_VALUES_EQUAL(sizeof(response), pos); - - UNIT_ASSERT_VALUES_EQUAL(YBUS_VERSION, response.GetVersionInternal()); - } - + pos += count; + } + + UNIT_ASSERT_VALUES_EQUAL(sizeof(response), pos); + + UNIT_ASSERT_VALUES_EQUAL(YBUS_VERSION, response.GetVersionInternal()); + } + struct TOnConnectionEventClient: public TExampleClient { - TTestSync Sync; - + TTestSync Sync; + ~TOnConnectionEventClient() override { - Session->Shutdown(); - } - + Session->Shutdown(); + } + void OnClientConnectionEvent(const TClientConnectionEvent& event) override { - if (Sync.Get() > 2) { - // Test OnClientConnectionEvent_Disconnect is broken. - // Sometimes reconnect happens during server shutdown - // when acceptor connections is still alive, and - // server connection is already closed - return; - } - - if (event.GetType() == TClientConnectionEvent::CONNECTED) { - Sync.WaitForAndIncrement(0); - } else if (event.GetType() == TClientConnectionEvent::DISCONNECTED) { - Sync.WaitForAndIncrement(2); - } - } + if (Sync.Get() > 2) { + // Test OnClientConnectionEvent_Disconnect is broken. + // Sometimes reconnect happens during server shutdown + // when acceptor connections is still alive, and + // server connection is already closed + return; + } + + if (event.GetType() == TClientConnectionEvent::CONNECTED) { + Sync.WaitForAndIncrement(0); + } else if (event.GetType() == TClientConnectionEvent::DISCONNECTED) { + Sync.WaitForAndIncrement(2); + } + } void OnError(TAutoPtr<TBusMessage>, EMessageStatus) override { // We do not check for message errors in this test. @@ -911,8 +911,8 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { void OnMessageSentOneWay(TAutoPtr<TBusMessage>) override { } - }; - + }; + struct TOnConnectionEventServer: public TExampleServer { TOnConnectionEventServer() : TExampleServer("TOnConnectionEventServer") @@ -929,39 +929,39 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { }; Y_UNIT_TEST(OnClientConnectionEvent_Shutdown) { - TObjectCountCheck objectCountCheck; - + TObjectCountCheck objectCountCheck; + TOnConnectionEventServer server; - - TOnConnectionEventClient client; - - TNetAddr addr("127.0.0.1", server.Session->GetActualListenPort()); - + + TOnConnectionEventClient client; + + TNetAddr addr("127.0.0.1", server.Session->GetActualListenPort()); + client.Session->SendMessageOneWay(new TExampleRequest(&client.Proto.RequestCount), &addr); - - client.Sync.WaitForAndIncrement(1); - - client.Session->Shutdown(); - - client.Sync.WaitForAndIncrement(3); - } - + + client.Sync.WaitForAndIncrement(1); + + client.Session->Shutdown(); + + client.Sync.WaitForAndIncrement(3); + } + Y_UNIT_TEST(OnClientConnectionEvent_Disconnect) { - TObjectCountCheck objectCountCheck; - + TObjectCountCheck objectCountCheck; + THolder<TOnConnectionEventServer> server(new TOnConnectionEventServer); - - TOnConnectionEventClient client; - TNetAddr addr("127.0.0.1", server->Session->GetActualListenPort()); - + + TOnConnectionEventClient client; + TNetAddr addr("127.0.0.1", server->Session->GetActualListenPort()); + client.Session->SendMessageOneWay(new TExampleRequest(&client.Proto.RequestCount), &addr); - - client.Sync.WaitForAndIncrement(1); - - server.Destroy(); - - client.Sync.WaitForAndIncrement(3); - } + + client.Sync.WaitForAndIncrement(1); + + server.Destroy(); + + client.Sync.WaitForAndIncrement(3); + } struct TServerForQuotaWake: public TExampleServer { TSystemEvent GoOn; @@ -1042,7 +1042,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { start = now; // TODO: properly check that server is blocked - } else if (start + TDuration::MilliSeconds(100) < now) { + } else if (start + TDuration::MilliSeconds(100) < now) { break; } } diff --git a/library/cpp/messagebus/test/ut/module_client_one_way_ut.cpp b/library/cpp/messagebus/test/ut/module_client_one_way_ut.cpp index 9c1224ada9..4083cf3b7b 100644 --- a/library/cpp/messagebus/test/ut/module_client_one_way_ut.cpp +++ b/library/cpp/messagebus/test/ut/module_client_one_way_ut.cpp @@ -1,143 +1,143 @@ #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/messagebus/test/helper/example.h> #include <library/cpp/messagebus/test/helper/message_handler_error.h> - + #include <library/cpp/messagebus/misc/test_sync.h> #include <library/cpp/messagebus/oldmodule/module.h> -using namespace NBus; -using namespace NBus::NTest; - +using namespace NBus; +using namespace NBus::NTest; + Y_UNIT_TEST_SUITE(ModuleClientOneWay) { - struct TTestServer: public TBusServerHandlerError { - TExampleProtocol Proto; - - TTestSync* const TestSync; - - TBusMessageQueuePtr Queue; - TBusServerSessionPtr ServerSession; - - TTestServer(TTestSync* testSync) - : TestSync(testSync) - { - Queue = CreateMessageQueue(); - ServerSession = TBusServerSession::Create(&Proto, this, TBusServerSessionConfig(), Queue); - } - + struct TTestServer: public TBusServerHandlerError { + TExampleProtocol Proto; + + TTestSync* const TestSync; + + TBusMessageQueuePtr Queue; + TBusServerSessionPtr ServerSession; + + TTestServer(TTestSync* testSync) + : TestSync(testSync) + { + Queue = CreateMessageQueue(); + ServerSession = TBusServerSession::Create(&Proto, this, TBusServerSessionConfig(), Queue); + } + void OnMessage(TOnMessageContext& context) override { - TestSync->WaitForAndIncrement(1); - context.ForgetRequest(); - } - }; - - struct TClientModule: public TBusModule { - TExampleProtocol Proto; - - TTestSync* const TestSync; - unsigned const Port; - - TBusClientSessionPtr ClientSession; - - TClientModule(TTestSync* testSync, unsigned port) - : TBusModule("m") - , TestSync(testSync) - , Port(port) + TestSync->WaitForAndIncrement(1); + context.ForgetRequest(); + } + }; + + struct TClientModule: public TBusModule { + TExampleProtocol Proto; + + TTestSync* const TestSync; + unsigned const Port; + + TBusClientSessionPtr ClientSession; + + TClientModule(TTestSync* testSync, unsigned port) + : TBusModule("m") + , TestSync(testSync) + , Port(port) { } - + TJobHandler Start(TBusJob* job, TBusMessage*) override { - TestSync->WaitForAndIncrement(0); - - job->SendOneWayTo(new TExampleRequest(&Proto.RequestCount), ClientSession.Get(), TNetAddr("localhost", Port)); - - return &TClientModule::Sent; - } - - TJobHandler Sent(TBusJob* job, TBusMessage*) { - TestSync->WaitForAndIncrement(2); - job->Cancel(MESSAGE_DONT_ASK); + TestSync->WaitForAndIncrement(0); + + job->SendOneWayTo(new TExampleRequest(&Proto.RequestCount), ClientSession.Get(), TNetAddr("localhost", Port)); + + return &TClientModule::Sent; + } + + TJobHandler Sent(TBusJob* job, TBusMessage*) { + TestSync->WaitForAndIncrement(2); + job->Cancel(MESSAGE_DONT_ASK); return nullptr; - } - + } + TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override { - ClientSession = CreateDefaultSource(queue, &Proto, TBusServerSessionConfig()); + ClientSession = CreateDefaultSource(queue, &Proto, TBusServerSessionConfig()); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(Simple) { - TTestSync testSync; - - TTestServer server(&testSync); - - TBusMessageQueuePtr queue = CreateMessageQueue(); - TClientModule clientModule(&testSync, server.ServerSession->GetActualListenPort()); - - clientModule.CreatePrivateSessions(queue.Get()); - clientModule.StartInput(); - - clientModule.StartJob(new TExampleRequest(&clientModule.Proto.StartCount)); - - testSync.WaitForAndIncrement(3); - - clientModule.Shutdown(); - } - - struct TSendErrorModule: public TBusModule { - TExampleProtocol Proto; - - TTestSync* const TestSync; - - TBusClientSessionPtr ClientSession; - - TSendErrorModule(TTestSync* testSync) - : TBusModule("m") - , TestSync(testSync) + TTestSync testSync; + + TTestServer server(&testSync); + + TBusMessageQueuePtr queue = CreateMessageQueue(); + TClientModule clientModule(&testSync, server.ServerSession->GetActualListenPort()); + + clientModule.CreatePrivateSessions(queue.Get()); + clientModule.StartInput(); + + clientModule.StartJob(new TExampleRequest(&clientModule.Proto.StartCount)); + + testSync.WaitForAndIncrement(3); + + clientModule.Shutdown(); + } + + struct TSendErrorModule: public TBusModule { + TExampleProtocol Proto; + + TTestSync* const TestSync; + + TBusClientSessionPtr ClientSession; + + TSendErrorModule(TTestSync* testSync) + : TBusModule("m") + , TestSync(testSync) { } - + TJobHandler Start(TBusJob* job, TBusMessage*) override { - TestSync->WaitForAndIncrement(0); - - job->SendOneWayTo(new TExampleRequest(&Proto.RequestCount), ClientSession.Get(), TNetAddr("localhost", 1)); - - return &TSendErrorModule::Sent; - } - - TJobHandler Sent(TBusJob* job, TBusMessage*) { - TestSync->WaitForAndIncrement(1); - job->Cancel(MESSAGE_DONT_ASK); + TestSync->WaitForAndIncrement(0); + + job->SendOneWayTo(new TExampleRequest(&Proto.RequestCount), ClientSession.Get(), TNetAddr("localhost", 1)); + + return &TSendErrorModule::Sent; + } + + TJobHandler Sent(TBusJob* job, TBusMessage*) { + TestSync->WaitForAndIncrement(1); + job->Cancel(MESSAGE_DONT_ASK); return nullptr; - } - + } + TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override { - TBusServerSessionConfig sessionConfig; - sessionConfig.ConnectTimeout = 1; - sessionConfig.SendTimeout = 1; - sessionConfig.TotalTimeout = 1; - sessionConfig.Secret.TimeoutPeriod = TDuration::MilliSeconds(1); - ClientSession = CreateDefaultSource(queue, &Proto, sessionConfig); + TBusServerSessionConfig sessionConfig; + sessionConfig.ConnectTimeout = 1; + sessionConfig.SendTimeout = 1; + sessionConfig.TotalTimeout = 1; + sessionConfig.Secret.TimeoutPeriod = TDuration::MilliSeconds(1); + ClientSession = CreateDefaultSource(queue, &Proto, sessionConfig); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(SendError) { - TTestSync testSync; - - TBusQueueConfig queueConfig; - queueConfig.NumWorkers = 5; - - TBusMessageQueuePtr queue = CreateMessageQueue(queueConfig); - TSendErrorModule clientModule(&testSync); - - clientModule.CreatePrivateSessions(queue.Get()); - clientModule.StartInput(); - - clientModule.StartJob(new TExampleRequest(&clientModule.Proto.StartCount)); - - testSync.WaitForAndIncrement(2); - - clientModule.Shutdown(); - } -} + TTestSync testSync; + + TBusQueueConfig queueConfig; + queueConfig.NumWorkers = 5; + + TBusMessageQueuePtr queue = CreateMessageQueue(queueConfig); + TSendErrorModule clientModule(&testSync); + + clientModule.CreatePrivateSessions(queue.Get()); + clientModule.StartInput(); + + clientModule.StartJob(new TExampleRequest(&clientModule.Proto.StartCount)); + + testSync.WaitForAndIncrement(2); + + clientModule.Shutdown(); + } +} diff --git a/library/cpp/messagebus/test/ut/module_client_ut.cpp b/library/cpp/messagebus/test/ut/module_client_ut.cpp index faffdbb625..ebfe185cc6 100644 --- a/library/cpp/messagebus/test/ut/module_client_ut.cpp +++ b/library/cpp/messagebus/test/ut/module_client_ut.cpp @@ -1,368 +1,368 @@ #include <library/cpp/testing/unittest/registar.h> - + #include "count_down_latch.h" -#include "moduletest.h" - +#include "moduletest.h" + #include <library/cpp/messagebus/test/helper/example.h> #include <library/cpp/messagebus/test/helper/example_module.h> #include <library/cpp/messagebus/test/helper/object_count_check.h> #include <library/cpp/messagebus/test/helper/wait_for.h> - + #include <library/cpp/messagebus/misc/test_sync.h> #include <library/cpp/messagebus/oldmodule/module.h> #include <util/generic/cast.h> #include <util/system/event.h> -using namespace NBus; -using namespace NBus::NTest; - -// helper class that cleans TBusJob instance, so job's destructor can -// be completed without assertion fail. -struct TJobGuard { +using namespace NBus; +using namespace NBus::NTest; + +// helper class that cleans TBusJob instance, so job's destructor can +// be completed without assertion fail. +struct TJobGuard { public: TJobGuard(NBus::TBusJob* job) : Job(job) { } - + ~TJobGuard() { Job->ClearAllMessageStates(); } - + private: NBus::TBusJob* Job; -}; - +}; + class TMessageOk: public NBus::TBusMessage { public: TMessageOk() : NBus::TBusMessage(1) { } -}; - +}; + class TMessageError: public NBus::TBusMessage { public: TMessageError() : NBus::TBusMessage(2) { } -}; - +}; + Y_UNIT_TEST_SUITE(BusJobTest) { -#if 0 +#if 0 Y_UNIT_TEST(TestPending) { - TObjectCountCheck objectCountCheck; - - TDupDetectModule module; - TBusJob job(&module, new TBusMessage(0)); - // Guard will clear the job if unit-assertion fails. - TJobGuard g(&job); - - NBus::TBusMessage* msg = new NBus::TBusMessage(1); - job.Send(msg, NULL); - NBus::TJobStateVec pending; - job.GetPending(&pending); - - UNIT_ASSERT_VALUES_EQUAL(pending.size(), 1u); - UNIT_ASSERT_EQUAL(msg, pending[0].Message); - } - + TObjectCountCheck objectCountCheck; + + TDupDetectModule module; + TBusJob job(&module, new TBusMessage(0)); + // Guard will clear the job if unit-assertion fails. + TJobGuard g(&job); + + NBus::TBusMessage* msg = new NBus::TBusMessage(1); + job.Send(msg, NULL); + NBus::TJobStateVec pending; + job.GetPending(&pending); + + UNIT_ASSERT_VALUES_EQUAL(pending.size(), 1u); + UNIT_ASSERT_EQUAL(msg, pending[0].Message); + } + Y_UNIT_TEST(TestCallReplyHandler) { - TObjectCountCheck objectCountCheck; - - TDupDetectModule module; - NBus::TBusJob job(&module, new NBus::TBusMessage(0)); - // Guard will clear the job if unit-assertion fails. - TJobGuard g(&job); - - NBus::TBusMessage* msgOk = new TMessageOk; - NBus::TBusMessage* msgError = new TMessageError; - job.Send(msgOk, NULL); - job.Send(msgError, NULL); - - UNIT_ASSERT_EQUAL(job.GetState<TMessageOk>(), NULL); - UNIT_ASSERT_EQUAL(job.GetState<TMessageError>(), NULL); - - NBus::TBusMessage* reply = new NBus::TBusMessage(0); - job.CallReplyHandler(NBus::MESSAGE_OK, msgOk, reply); - job.CallReplyHandler(NBus::MESSAGE_TIMEOUT, msgError, NULL); - - UNIT_ASSERT_UNEQUAL(job.GetState<TMessageOk>(), NULL); - UNIT_ASSERT_UNEQUAL(job.GetState<TMessageError>(), NULL); - - UNIT_ASSERT_VALUES_EQUAL(job.GetStatus<TMessageError>(), NBus::MESSAGE_TIMEOUT); - UNIT_ASSERT_EQUAL(job.GetState<TMessageError>()->Status, NBus::MESSAGE_TIMEOUT); - - UNIT_ASSERT_VALUES_EQUAL(job.GetStatus<TMessageOk>(), NBus::MESSAGE_OK); - UNIT_ASSERT_EQUAL(job.GetState<TMessageOk>()->Reply, reply); - } -#endif - - struct TParallelOnReplyModule : TExampleClientModule { - TNetAddr ServerAddr; - - TCountDownLatch RepliesLatch; - - TParallelOnReplyModule(const TNetAddr& serverAddr) - : ServerAddr(serverAddr) - , RepliesLatch(2) + TObjectCountCheck objectCountCheck; + + TDupDetectModule module; + NBus::TBusJob job(&module, new NBus::TBusMessage(0)); + // Guard will clear the job if unit-assertion fails. + TJobGuard g(&job); + + NBus::TBusMessage* msgOk = new TMessageOk; + NBus::TBusMessage* msgError = new TMessageError; + job.Send(msgOk, NULL); + job.Send(msgError, NULL); + + UNIT_ASSERT_EQUAL(job.GetState<TMessageOk>(), NULL); + UNIT_ASSERT_EQUAL(job.GetState<TMessageError>(), NULL); + + NBus::TBusMessage* reply = new NBus::TBusMessage(0); + job.CallReplyHandler(NBus::MESSAGE_OK, msgOk, reply); + job.CallReplyHandler(NBus::MESSAGE_TIMEOUT, msgError, NULL); + + UNIT_ASSERT_UNEQUAL(job.GetState<TMessageOk>(), NULL); + UNIT_ASSERT_UNEQUAL(job.GetState<TMessageError>(), NULL); + + UNIT_ASSERT_VALUES_EQUAL(job.GetStatus<TMessageError>(), NBus::MESSAGE_TIMEOUT); + UNIT_ASSERT_EQUAL(job.GetState<TMessageError>()->Status, NBus::MESSAGE_TIMEOUT); + + UNIT_ASSERT_VALUES_EQUAL(job.GetStatus<TMessageOk>(), NBus::MESSAGE_OK); + UNIT_ASSERT_EQUAL(job.GetState<TMessageOk>()->Reply, reply); + } +#endif + + struct TParallelOnReplyModule : TExampleClientModule { + TNetAddr ServerAddr; + + TCountDownLatch RepliesLatch; + + TParallelOnReplyModule(const TNetAddr& serverAddr) + : ServerAddr(serverAddr) + , RepliesLatch(2) { } - + TJobHandler Start(TBusJob* job, TBusMessage* mess) override { Y_UNUSED(mess); - job->Send(new TExampleRequest(&Proto.RequestCount), Source, TReplyHandler(&TParallelOnReplyModule::ReplyHandler), 0, ServerAddr); - return &TParallelOnReplyModule::HandleReplies; - } - - void ReplyHandler(TBusJob*, EMessageStatus status, TBusMessage* mess, TBusMessage* reply) { + job->Send(new TExampleRequest(&Proto.RequestCount), Source, TReplyHandler(&TParallelOnReplyModule::ReplyHandler), 0, ServerAddr); + return &TParallelOnReplyModule::HandleReplies; + } + + void ReplyHandler(TBusJob*, EMessageStatus status, TBusMessage* mess, TBusMessage* reply) { Y_UNUSED(mess); Y_UNUSED(reply); Y_VERIFY(status == MESSAGE_OK, "failed to get reply: %s", ToCString(status)); - } - - TJobHandler HandleReplies(TBusJob* job, TBusMessage* mess) { + } + + TJobHandler HandleReplies(TBusJob* job, TBusMessage* mess) { Y_UNUSED(mess); - RepliesLatch.CountDown(); + RepliesLatch.CountDown(); Y_VERIFY(RepliesLatch.Await(TDuration::Seconds(10)), "failed to get answers"); - job->Cancel(MESSAGE_UNKNOWN); + job->Cancel(MESSAGE_UNKNOWN); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(TestReplyHandlerCalledInParallel) { - TObjectCountCheck objectCountCheck; - - TExampleServer server; - - TExampleProtocol proto; - - TBusQueueConfig config; - config.NumWorkers = 5; - - TParallelOnReplyModule module(server.GetActualListenAddr()); - module.StartModule(); - - module.StartJob(new TExampleRequest(&proto.StartCount)); - module.StartJob(new TExampleRequest(&proto.StartCount)); - - UNIT_ASSERT(module.RepliesLatch.Await(TDuration::Seconds(10))); - - module.Shutdown(); - } - - struct TErrorHandlerCheckerModule : TExampleModule { - TNetAddr ServerAddr; - - TBusClientSessionPtr Source; - - TCountDownLatch GotReplyLatch; - - TBusMessage* SentMessage; - - TErrorHandlerCheckerModule() - : ServerAddr("localhost", 17) - , GotReplyLatch(2) - , SentMessage() + TObjectCountCheck objectCountCheck; + + TExampleServer server; + + TExampleProtocol proto; + + TBusQueueConfig config; + config.NumWorkers = 5; + + TParallelOnReplyModule module(server.GetActualListenAddr()); + module.StartModule(); + + module.StartJob(new TExampleRequest(&proto.StartCount)); + module.StartJob(new TExampleRequest(&proto.StartCount)); + + UNIT_ASSERT(module.RepliesLatch.Await(TDuration::Seconds(10))); + + module.Shutdown(); + } + + struct TErrorHandlerCheckerModule : TExampleModule { + TNetAddr ServerAddr; + + TBusClientSessionPtr Source; + + TCountDownLatch GotReplyLatch; + + TBusMessage* SentMessage; + + TErrorHandlerCheckerModule() + : ServerAddr("localhost", 17) + , GotReplyLatch(2) + , SentMessage() { } - + TJobHandler Start(TBusJob* job, TBusMessage* mess) override { Y_UNUSED(mess); - TExampleRequest* message = new TExampleRequest(&Proto.RequestCount); - job->Send(message, Source, TReplyHandler(&TErrorHandlerCheckerModule::ReplyHandler), 0, ServerAddr); - SentMessage = message; - return &TErrorHandlerCheckerModule::HandleReplies; - } - - void ReplyHandler(TBusJob*, EMessageStatus status, TBusMessage* req, TBusMessage* resp) { + TExampleRequest* message = new TExampleRequest(&Proto.RequestCount); + job->Send(message, Source, TReplyHandler(&TErrorHandlerCheckerModule::ReplyHandler), 0, ServerAddr); + SentMessage = message; + return &TErrorHandlerCheckerModule::HandleReplies; + } + + void ReplyHandler(TBusJob*, EMessageStatus status, TBusMessage* req, TBusMessage* resp) { Y_VERIFY(status == MESSAGE_CONNECT_FAILED || status == MESSAGE_TIMEOUT, "got wrong status: %s", ToString(status).data()); Y_VERIFY(req == SentMessage, "checking request"); Y_VERIFY(resp == nullptr, "checking response"); - GotReplyLatch.CountDown(); - } - - TJobHandler HandleReplies(TBusJob* job, TBusMessage* mess) { + GotReplyLatch.CountDown(); + } + + TJobHandler HandleReplies(TBusJob* job, TBusMessage* mess) { Y_UNUSED(mess); - job->Cancel(MESSAGE_UNKNOWN); - GotReplyLatch.CountDown(); + job->Cancel(MESSAGE_UNKNOWN); + GotReplyLatch.CountDown(); return nullptr; - } - + } + TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override { - TBusClientSessionConfig sessionConfig; - sessionConfig.SendTimeout = 1; // TODO: allow 0 - sessionConfig.Secret.TimeoutPeriod = TDuration::MilliSeconds(10); - Source = CreateDefaultSource(queue, &Proto, sessionConfig); - Source->RegisterService("localhost"); + TBusClientSessionConfig sessionConfig; + sessionConfig.SendTimeout = 1; // TODO: allow 0 + sessionConfig.Secret.TimeoutPeriod = TDuration::MilliSeconds(10); + Source = CreateDefaultSource(queue, &Proto, sessionConfig); + Source->RegisterService("localhost"); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(ErrorHandler) { - TExampleProtocol proto; - - TBusQueueConfig config; - config.NumWorkers = 5; - - TErrorHandlerCheckerModule module; - - TBusModuleConfig moduleConfig; - moduleConfig.Secret.SchedulePeriod = TDuration::MilliSeconds(10); - module.SetConfig(moduleConfig); - - module.StartModule(); - - module.StartJob(new TExampleRequest(&proto.StartCount)); - - module.GotReplyLatch.Await(); - - module.Shutdown(); - } - + TExampleProtocol proto; + + TBusQueueConfig config; + config.NumWorkers = 5; + + TErrorHandlerCheckerModule module; + + TBusModuleConfig moduleConfig; + moduleConfig.Secret.SchedulePeriod = TDuration::MilliSeconds(10); + module.SetConfig(moduleConfig); + + module.StartModule(); + + module.StartJob(new TExampleRequest(&proto.StartCount)); + + module.GotReplyLatch.Await(); + + module.Shutdown(); + } + struct TSlowReplyServer: public TBusServerHandlerError { - TTestSync* const TestSync; - TBusMessageQueuePtr Bus; - TBusServerSessionPtr ServerSession; - TExampleProtocol Proto; - - TAtomic OnMessageCount; - - TSlowReplyServer(TTestSync* testSync) - : TestSync(testSync) - , OnMessageCount(0) - { - Bus = CreateMessageQueue("TSlowReplyServer"); - TBusServerSessionConfig sessionConfig; - ServerSession = TBusServerSession::Create(&Proto, this, sessionConfig, Bus); - } - + TTestSync* const TestSync; + TBusMessageQueuePtr Bus; + TBusServerSessionPtr ServerSession; + TExampleProtocol Proto; + + TAtomic OnMessageCount; + + TSlowReplyServer(TTestSync* testSync) + : TestSync(testSync) + , OnMessageCount(0) + { + Bus = CreateMessageQueue("TSlowReplyServer"); + TBusServerSessionConfig sessionConfig; + ServerSession = TBusServerSession::Create(&Proto, this, sessionConfig, Bus); + } + void OnMessage(TOnMessageContext& req) override { - if (AtomicIncrement(OnMessageCount) == 1) { - TestSync->WaitForAndIncrement(0); - } - TAutoPtr<TBusMessage> response(new TExampleResponse(&Proto.ResponseCount)); - req.SendReplyMove(response); - } - }; - + if (AtomicIncrement(OnMessageCount) == 1) { + TestSync->WaitForAndIncrement(0); + } + TAutoPtr<TBusMessage> response(new TExampleResponse(&Proto.ResponseCount)); + req.SendReplyMove(response); + } + }; + struct TModuleThatSendsReplyEarly: public TExampleClientModule { - TTestSync* const TestSync; - const unsigned ServerPort; - - TBusServerSessionPtr ServerSession; - TAtomic ReplyCount; - - TModuleThatSendsReplyEarly(TTestSync* testSync, unsigned serverPort) - : TestSync(testSync) - , ServerPort(serverPort) + TTestSync* const TestSync; + const unsigned ServerPort; + + TBusServerSessionPtr ServerSession; + TAtomic ReplyCount; + + TModuleThatSendsReplyEarly(TTestSync* testSync, unsigned serverPort) + : TestSync(testSync) + , ServerPort(serverPort) , ServerSession(nullptr) - , ReplyCount(0) + , ReplyCount(0) { } - + TJobHandler Start(TBusJob* job, TBusMessage* mess) override { Y_UNUSED(mess); - for (unsigned i = 0; i < 2; ++i) { - job->Send( - new TExampleRequest(&Proto.RequestCount), - Source, - TReplyHandler(&TModuleThatSendsReplyEarly::ReplyHandler), - 0, - TNetAddr("127.0.0.1", ServerPort)); - } - return &TModuleThatSendsReplyEarly::HandleReplies; - } - - void ReplyHandler(TBusJob* job, EMessageStatus status, TBusMessage* mess, TBusMessage* reply) { + for (unsigned i = 0; i < 2; ++i) { + job->Send( + new TExampleRequest(&Proto.RequestCount), + Source, + TReplyHandler(&TModuleThatSendsReplyEarly::ReplyHandler), + 0, + TNetAddr("127.0.0.1", ServerPort)); + } + return &TModuleThatSendsReplyEarly::HandleReplies; + } + + void ReplyHandler(TBusJob* job, EMessageStatus status, TBusMessage* mess, TBusMessage* reply) { Y_UNUSED(mess); Y_UNUSED(reply); Y_VERIFY(status == MESSAGE_OK, "failed to get reply"); - if (AtomicIncrement(ReplyCount) == 1) { - TestSync->WaitForAndIncrement(1); - job->SendReply(new TExampleResponse(&Proto.ResponseCount)); - } else { - TestSync->WaitForAndIncrement(3); - } - } - - TJobHandler HandleReplies(TBusJob* job, TBusMessage* mess) { + if (AtomicIncrement(ReplyCount) == 1) { + TestSync->WaitForAndIncrement(1); + job->SendReply(new TExampleResponse(&Proto.ResponseCount)); + } else { + TestSync->WaitForAndIncrement(3); + } + } + + TJobHandler HandleReplies(TBusJob* job, TBusMessage* mess) { Y_UNUSED(mess); - job->Cancel(MESSAGE_UNKNOWN); + job->Cancel(MESSAGE_UNKNOWN); return nullptr; - } - + } + TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override { - TExampleClientModule::CreateExtSession(queue); - TBusServerSessionConfig sessionConfig; - return ServerSession = CreateDefaultDestination(queue, &Proto, sessionConfig); - } - }; - + TExampleClientModule::CreateExtSession(queue); + TBusServerSessionConfig sessionConfig; + return ServerSession = CreateDefaultDestination(queue, &Proto, sessionConfig); + } + }; + Y_UNIT_TEST(SendReplyCalledBeforeAllRepliesReceived) { - TTestSync testSync; - - TSlowReplyServer slowReplyServer(&testSync); - - TModuleThatSendsReplyEarly module(&testSync, slowReplyServer.ServerSession->GetActualListenPort()); - module.StartModule(); - - TExampleClient client; - TNetAddr addr("127.0.0.1", module.ServerSession->GetActualListenPort()); - client.SendMessagesWaitReplies(1, &addr); - - testSync.WaitForAndIncrement(2); - - module.Shutdown(); - } - + TTestSync testSync; + + TSlowReplyServer slowReplyServer(&testSync); + + TModuleThatSendsReplyEarly module(&testSync, slowReplyServer.ServerSession->GetActualListenPort()); + module.StartModule(); + + TExampleClient client; + TNetAddr addr("127.0.0.1", module.ServerSession->GetActualListenPort()); + client.SendMessagesWaitReplies(1, &addr); + + testSync.WaitForAndIncrement(2); + + module.Shutdown(); + } + struct TShutdownCalledBeforeReplyReceivedModule: public TExampleClientModule { - unsigned ServerPort; - - TTestSync TestSync; - - TShutdownCalledBeforeReplyReceivedModule(unsigned serverPort) - : ServerPort(serverPort) + unsigned ServerPort; + + TTestSync TestSync; + + TShutdownCalledBeforeReplyReceivedModule(unsigned serverPort) + : ServerPort(serverPort) { } - + TJobHandler Start(TBusJob* job, TBusMessage*) override { - TestSync.CheckAndIncrement(0); - - job->Send(new TExampleRequest(&Proto.RequestCount), Source, + TestSync.CheckAndIncrement(0); + + job->Send(new TExampleRequest(&Proto.RequestCount), Source, TReplyHandler(&TShutdownCalledBeforeReplyReceivedModule::HandleReply), 0, TNetAddr("localhost", ServerPort)); - return &TShutdownCalledBeforeReplyReceivedModule::End; - } - - void HandleReply(TBusJob*, EMessageStatus status, TBusMessage*, TBusMessage*) { + return &TShutdownCalledBeforeReplyReceivedModule::End; + } + + void HandleReply(TBusJob*, EMessageStatus status, TBusMessage*, TBusMessage*) { Y_VERIFY(status == MESSAGE_SHUTDOWN, "got %s", ToCString(status)); - TestSync.CheckAndIncrement(1); - } - - TJobHandler End(TBusJob* job, TBusMessage*) { - TestSync.CheckAndIncrement(2); - job->Cancel(MESSAGE_SHUTDOWN); + TestSync.CheckAndIncrement(1); + } + + TJobHandler End(TBusJob* job, TBusMessage*) { + TestSync.CheckAndIncrement(2); + job->Cancel(MESSAGE_SHUTDOWN); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(ShutdownCalledBeforeReplyReceived) { - TExampleServer server; - server.ForgetRequest = true; - - TShutdownCalledBeforeReplyReceivedModule module(server.GetActualListenPort()); - - module.StartModule(); - - module.StartJob(new TExampleRequest(&module.Proto.RequestCount)); - - server.TestSync.WaitFor(1); - - module.Shutdown(); - - module.TestSync.CheckAndIncrement(3); - } -} + TExampleServer server; + server.ForgetRequest = true; + + TShutdownCalledBeforeReplyReceivedModule module(server.GetActualListenPort()); + + module.StartModule(); + + module.StartJob(new TExampleRequest(&module.Proto.RequestCount)); + + server.TestSync.WaitFor(1); + + module.Shutdown(); + + module.TestSync.CheckAndIncrement(3); + } +} diff --git a/library/cpp/messagebus/test/ut/module_server_ut.cpp b/library/cpp/messagebus/test/ut/module_server_ut.cpp index 38f3fcc4ed..88fe1dd9b6 100644 --- a/library/cpp/messagebus/test/ut/module_server_ut.cpp +++ b/library/cpp/messagebus/test/ut/module_server_ut.cpp @@ -1,8 +1,8 @@ #include <library/cpp/testing/unittest/registar.h> - + #include "count_down_latch.h" -#include "moduletest.h" - +#include "moduletest.h" + #include <library/cpp/messagebus/test/helper/example.h> #include <library/cpp/messagebus/test/helper/example_module.h> #include <library/cpp/messagebus/test/helper/object_count_check.h> @@ -12,108 +12,108 @@ #include <util/generic/cast.h> -using namespace NBus; -using namespace NBus::NTest; - +using namespace NBus; +using namespace NBus::NTest; + Y_UNIT_TEST_SUITE(ModuleServerTests) { Y_UNIT_TEST(TestModule) { - TObjectCountCheck objectCountCheck; - - /// create or get instance of message queue, need one per application - TBusMessageQueuePtr bus(CreateMessageQueue()); + TObjectCountCheck objectCountCheck; + + /// create or get instance of message queue, need one per application + TBusMessageQueuePtr bus(CreateMessageQueue()); THostInfoHandler hostHandler(bus.Get()); - TDupDetectModule module(hostHandler.GetActualListenAddr()); - bool success; - success = module.Init(bus.Get()); - UNIT_ASSERT_C(success, "failed to initialize dupdetect module"); - - success = module.StartInput(); - UNIT_ASSERT_C(success, "failed to start dupdetect module"); - - TDupDetectHandler dupHandler(module.ListenAddr, bus.Get()); - dupHandler.Work(); - - UNIT_WAIT_FOR(dupHandler.NumMessages == dupHandler.NumReplies); - - module.Shutdown(); - dupHandler.DupDetect->Shutdown(); - } - + TDupDetectModule module(hostHandler.GetActualListenAddr()); + bool success; + success = module.Init(bus.Get()); + UNIT_ASSERT_C(success, "failed to initialize dupdetect module"); + + success = module.StartInput(); + UNIT_ASSERT_C(success, "failed to start dupdetect module"); + + TDupDetectHandler dupHandler(module.ListenAddr, bus.Get()); + dupHandler.Work(); + + UNIT_WAIT_FOR(dupHandler.NumMessages == dupHandler.NumReplies); + + module.Shutdown(); + dupHandler.DupDetect->Shutdown(); + } + struct TParallelOnMessageModule: public TExampleServerModule { - TCountDownLatch WaitTwoRequestsLatch; - - TParallelOnMessageModule() - : WaitTwoRequestsLatch(2) + TCountDownLatch WaitTwoRequestsLatch; + + TParallelOnMessageModule() + : WaitTwoRequestsLatch(2) { } - + TJobHandler Start(TBusJob* job, TBusMessage* mess) override { - WaitTwoRequestsLatch.CountDown(); + WaitTwoRequestsLatch.CountDown(); Y_VERIFY(WaitTwoRequestsLatch.Await(TDuration::Seconds(5)), "oops"); - - VerifyDynamicCast<TExampleRequest*>(mess); - - job->SendReply(new TExampleResponse(&Proto.ResponseCount)); + + VerifyDynamicCast<TExampleRequest*>(mess); + + job->SendReply(new TExampleResponse(&Proto.ResponseCount)); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(TestOnMessageHandlerCalledInParallel) { - TObjectCountCheck objectCountCheck; - - TBusQueueConfig config; - config.NumWorkers = 5; - - TParallelOnMessageModule module; - module.StartModule(); - - TExampleClient client; - - client.SendMessagesWaitReplies(2, module.ServerAddr); - - module.Shutdown(); - } - - struct TDelayReplyServer: public TExampleServerModule { + TObjectCountCheck objectCountCheck; + + TBusQueueConfig config; + config.NumWorkers = 5; + + TParallelOnMessageModule module; + module.StartModule(); + + TExampleClient client; + + client.SendMessagesWaitReplies(2, module.ServerAddr); + + module.Shutdown(); + } + + struct TDelayReplyServer: public TExampleServerModule { TSystemEvent MessageReceivedEvent; TSystemEvent ClientDiedEvent; - + TJobHandler Start(TBusJob* job, TBusMessage* mess) override { Y_UNUSED(mess); - - MessageReceivedEvent.Signal(); - + + MessageReceivedEvent.Signal(); + Y_VERIFY(ClientDiedEvent.WaitT(TDuration::Seconds(5)), "oops"); - - job->SendReply(new TExampleResponse(&Proto.ResponseCount)); + + job->SendReply(new TExampleResponse(&Proto.ResponseCount)); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(TestReplyCalledAfterClientDisconnected) { - TObjectCountCheck objectCountCheck; - - TBusQueueConfig config; - config.NumWorkers = 5; - - TDelayReplyServer server; - server.StartModule(); - - THolder<TExampleClient> client(new TExampleClient); - - client->SendMessages(1, server.ServerAddr); - - UNIT_ASSERT(server.MessageReceivedEvent.WaitT(TDuration::Seconds(5))); - - UNIT_ASSERT_VALUES_EQUAL(1, server.GetModuleSessionInFlight()); - - client.Destroy(); - - server.ClientDiedEvent.Signal(); - - // wait until all server message are delivered - UNIT_WAIT_FOR(0 == server.GetModuleSessionInFlight()); - - server.Shutdown(); - } -} + TObjectCountCheck objectCountCheck; + + TBusQueueConfig config; + config.NumWorkers = 5; + + TDelayReplyServer server; + server.StartModule(); + + THolder<TExampleClient> client(new TExampleClient); + + client->SendMessages(1, server.ServerAddr); + + UNIT_ASSERT(server.MessageReceivedEvent.WaitT(TDuration::Seconds(5))); + + UNIT_ASSERT_VALUES_EQUAL(1, server.GetModuleSessionInFlight()); + + client.Destroy(); + + server.ClientDiedEvent.Signal(); + + // wait until all server message are delivered + UNIT_WAIT_FOR(0 == server.GetModuleSessionInFlight()); + + server.Shutdown(); + } +} diff --git a/library/cpp/messagebus/test/ut/moduletest.h b/library/cpp/messagebus/test/ut/moduletest.h index 0f9834d9ff..d5da72c0cb 100644 --- a/library/cpp/messagebus/test/ut/moduletest.h +++ b/library/cpp/messagebus/test/ut/moduletest.h @@ -7,10 +7,10 @@ #include <library/cpp/messagebus/test/helper/alloc_counter.h> #include <library/cpp/messagebus/test/helper/example.h> #include <library/cpp/messagebus/test/helper/message_handler_error.h> - + #include <library/cpp/messagebus/ybus.h> #include <library/cpp/messagebus/oldmodule/module.h> - + namespace NBus { namespace NTest { using namespace std; @@ -71,7 +71,7 @@ namespace NBus { /// deserialized TBusData into new instance of the message TAutoPtr<TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) override { Y_UNUSED(payload); - + if (messageType == TYPE_HOSTINFOREQUEST) { return new THostInfoMessage(MESSAGE_CREATE_UNINITIALIZED); } else if (messageType == TYPE_HOSTINFORESPONSE) { @@ -100,7 +100,7 @@ namespace NBus { mess.SendReplyMove(reply); } - + TNetAddr GetActualListenAddr() { return TNetAddr("localhost", Session->GetActualListenPort()); } @@ -110,7 +110,7 @@ namespace NBus { /// \brief DupDetect handler (should convert it to module too) struct TDupDetectHandler: public TBusClientHandlerError { TNetAddr ServerAddr; - + TBusClientSessionPtr DupDetect; TBusClientSessionConfig DupDetectConfig; TExampleProtocol DupDetectProto; @@ -147,7 +147,7 @@ namespace NBus { struct TDupDetectModule: public TBusModule { TNetAddr HostInfoAddr; - + TBusClientSessionPtr HostInfoClientSession; TBusClientSessionConfig HostInfoConfig; THostInfoProtocol HostInfoProto; @@ -162,7 +162,7 @@ namespace NBus { , HostInfoAddr(hostInfoAddr) { } - + bool Init(TBusMessageQueue* queue) { HostInfoClientSession = CreateDefaultSource(*queue, &HostInfoProto, HostInfoConfig); HostInfoClientSession->RegisterService("localhost"); @@ -174,7 +174,7 @@ namespace NBus { TBusServerSessionPtr session = CreateDefaultDestination(queue, &DupDetectProto, DupDetectConfig); ListenAddr = TNetAddr("localhost", session->GetActualListenPort()); - + return session; } diff --git a/library/cpp/messagebus/test/ut/one_way_ut.cpp b/library/cpp/messagebus/test/ut/one_way_ut.cpp index 7a907cc620..9c21227e2b 100644 --- a/library/cpp/messagebus/test/ut/one_way_ut.cpp +++ b/library/cpp/messagebus/test/ut/one_way_ut.cpp @@ -32,33 +32,33 @@ #include <library/cpp/messagebus/test/helper/wait_for.h> #include <library/cpp/messagebus/ybus.h> - + using namespace std; -using namespace NBus; -using namespace NBus::NPrivate; -using namespace NBus::NTest; +using namespace NBus; +using namespace NBus::NPrivate; +using namespace NBus::NTest; //////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////// /// \brief Reply-less client and handler struct NullClient : TBusClientHandlerError { - TNetAddr ServerAddr; - - TBusMessageQueuePtr Queue; - TBusClientSessionPtr Session; - TExampleProtocol Proto; + TNetAddr ServerAddr; + + TBusMessageQueuePtr Queue; + TBusClientSessionPtr Session; + TExampleProtocol Proto; /// constructor creates instances of protocol and session - NullClient(const TNetAddr& serverAddr, const TBusClientSessionConfig& sessionConfig = TBusClientSessionConfig()) - : ServerAddr(serverAddr) - { - UNIT_ASSERT(serverAddr.GetPort() > 0); + NullClient(const TNetAddr& serverAddr, const TBusClientSessionConfig& sessionConfig = TBusClientSessionConfig()) + : ServerAddr(serverAddr) + { + UNIT_ASSERT(serverAddr.GetPort() > 0); /// create or get instance of message queue, need one per application Queue = CreateMessageQueue(); /// register source/client session - Session = TBusClientSession::Create(&Proto, this, sessionConfig, Queue); + Session = TBusClientSession::Create(&Proto, this, sessionConfig, Queue); /// register service, announce to clients via LocatorService Session->RegisterService("localhost"); @@ -74,8 +74,8 @@ struct NullClient : TBusClientHandlerError { for (int i = 0; i < batch; i++) { TExampleRequest* mess = new TExampleRequest(&Proto.RequestCount); - mess->Data = "TADA"; - Session->SendMessageOneWay(mess, &ServerAddr); + mess->Data = "TADA"; + Session->SendMessageOneWay(mess, &ServerAddr); } } @@ -85,12 +85,12 @@ struct NullClient : TBusClientHandlerError { ///////////////////////////////////////////////////////////////////// /// \brief Reply-less server and handler -class NullServer: public TBusServerHandlerError { +class NullServer: public TBusServerHandlerError { public: /// session object to maintian - TBusMessageQueuePtr Queue; - TBusServerSessionPtr Session; - TExampleProtocol Proto; + TBusMessageQueuePtr Queue; + TBusServerSessionPtr Session; + TExampleProtocol Proto; public: TAtomic NumMessages; @@ -102,8 +102,8 @@ public: Queue = CreateMessageQueue(); /// register destination session - TBusServerSessionConfig sessionConfig; - Session = TBusServerSession::Create(&Proto, this, sessionConfig, Queue); + TBusServerSessionConfig sessionConfig; + Session = TBusServerSession::Create(&Proto, this, sessionConfig, Queue); } ~NullServer() override { @@ -117,7 +117,7 @@ public: Y_ASSERT(fmess->Data == "TADA"); /// tell session to forget this message and never expect any reply - mess.ForgetRequest(); + mess.ForgetRequest(); AtomicIncrement(NumMessages); } @@ -131,125 +131,125 @@ public: Y_UNIT_TEST_SUITE(TMessageBusTests_OneWay) { Y_UNIT_TEST(Simple) { - TObjectCountCheck objectCountCheck; - - NullServer server; - NullClient client(TNetAddr("localhost", server.Session->GetActualListenPort())); - - client.Work(); - - // wait until all client message are delivered + TObjectCountCheck objectCountCheck; + + NullServer server; + NullClient client(TNetAddr("localhost", server.Session->GetActualListenPort())); + + client.Work(); + + // wait until all client message are delivered UNIT_WAIT_FOR(AtomicGet(server.NumMessages) == 10); - - // assert correct number of messages + + // assert correct number of messages UNIT_ASSERT_VALUES_EQUAL(AtomicGet(server.NumMessages), 10); - UNIT_ASSERT_VALUES_EQUAL(server.Session->GetInFlight(), 0); - UNIT_ASSERT_VALUES_EQUAL(client.Session->GetInFlight(), 0); - } - - struct TMessageTooLargeClient: public NullClient { + UNIT_ASSERT_VALUES_EQUAL(server.Session->GetInFlight(), 0); + UNIT_ASSERT_VALUES_EQUAL(client.Session->GetInFlight(), 0); + } + + struct TMessageTooLargeClient: public NullClient { TSystemEvent GotTooLarge; - - TBusClientSessionConfig Config() { - TBusClientSessionConfig r; - r.MaxMessageSize = 1; - return r; - } - - TMessageTooLargeClient(unsigned port) - : NullClient(TNetAddr("localhost", port), Config()) + + TBusClientSessionConfig Config() { + TBusClientSessionConfig r; + r.MaxMessageSize = 1; + return r; + } + + TMessageTooLargeClient(unsigned port) + : NullClient(TNetAddr("localhost", port), Config()) { } - + ~TMessageTooLargeClient() override { - Session->Shutdown(); - } - + Session->Shutdown(); + } + void OnError(TAutoPtr<TBusMessage> mess, EMessageStatus status) override { Y_UNUSED(mess); - + Y_VERIFY(status == MESSAGE_MESSAGE_TOO_LARGE, "wrong status: %s", ToCString(status)); - - GotTooLarge.Signal(); - } - }; - + + GotTooLarge.Signal(); + } + }; + Y_UNIT_TEST(MessageTooLargeOnClient) { - TObjectCountCheck objectCountCheck; - - NullServer server; - - TMessageTooLargeClient client(server.Session->GetActualListenPort()); - - EMessageStatus ok = client.Session->SendMessageOneWayMove(new TExampleRequest(&client.Proto.RequestCount), &client.ServerAddr); - UNIT_ASSERT_VALUES_EQUAL(MESSAGE_OK, ok); - - client.GotTooLarge.WaitI(); - } - + TObjectCountCheck objectCountCheck; + + NullServer server; + + TMessageTooLargeClient client(server.Session->GetActualListenPort()); + + EMessageStatus ok = client.Session->SendMessageOneWayMove(new TExampleRequest(&client.Proto.RequestCount), &client.ServerAddr); + UNIT_ASSERT_VALUES_EQUAL(MESSAGE_OK, ok); + + client.GotTooLarge.WaitI(); + } + struct TCheckTimeoutClient: public NullClient { ~TCheckTimeoutClient() override { - Session->Shutdown(); - } - - static TBusClientSessionConfig SessionConfig() { - TBusClientSessionConfig sessionConfig; - sessionConfig.SendTimeout = 1; - sessionConfig.ConnectTimeout = 1; + Session->Shutdown(); + } + + static TBusClientSessionConfig SessionConfig() { + TBusClientSessionConfig sessionConfig; + sessionConfig.SendTimeout = 1; + sessionConfig.ConnectTimeout = 1; sessionConfig.Secret.TimeoutPeriod = TDuration::MilliSeconds(10); - return sessionConfig; - } - + return sessionConfig; + } + TCheckTimeoutClient(const TNetAddr& serverAddr) : NullClient(serverAddr, SessionConfig()) { } - + TSystemEvent GotError; - - /// message that could not be delivered + + /// message that could not be delivered void OnError(TAutoPtr<TBusMessage> mess, EMessageStatus status) override { Y_UNUSED(mess); Y_UNUSED(status); // TODO: check status - - GotError.Signal(); - } - }; - + + GotError.Signal(); + } + }; + Y_UNIT_TEST(SendTimeout_Callback_NoServer) { - TObjectCountCheck objectCountCheck; - - TCheckTimeoutClient client(TNetAddr("localhost", 17)); - - EMessageStatus ok = client.Session->SendMessageOneWay(new TExampleRequest(&client.Proto.RequestCount), &client.ServerAddr); - UNIT_ASSERT_EQUAL(ok, MESSAGE_OK); - - client.GotError.WaitI(); - } - + TObjectCountCheck objectCountCheck; + + TCheckTimeoutClient client(TNetAddr("localhost", 17)); + + EMessageStatus ok = client.Session->SendMessageOneWay(new TExampleRequest(&client.Proto.RequestCount), &client.ServerAddr); + UNIT_ASSERT_EQUAL(ok, MESSAGE_OK); + + client.GotError.WaitI(); + } + Y_UNIT_TEST(SendTimeout_Callback_HangingServer) { - THangingServer server; - - TObjectCountCheck objectCountCheck; - - TCheckTimeoutClient client(TNetAddr("localhost", server.GetPort())); - - bool first = true; - for (;;) { - EMessageStatus ok = client.Session->SendMessageOneWayMove(new TExampleRequest(&client.Proto.RequestCount), &client.ServerAddr); - if (ok == MESSAGE_BUSY) { - UNIT_ASSERT(!first); - break; - } - UNIT_ASSERT_VALUES_EQUAL(ok, MESSAGE_OK); - first = false; - } - + THangingServer server; + + TObjectCountCheck objectCountCheck; + + TCheckTimeoutClient client(TNetAddr("localhost", server.GetPort())); + + bool first = true; + for (;;) { + EMessageStatus ok = client.Session->SendMessageOneWayMove(new TExampleRequest(&client.Proto.RequestCount), &client.ServerAddr); + if (ok == MESSAGE_BUSY) { + UNIT_ASSERT(!first); + break; + } + UNIT_ASSERT_VALUES_EQUAL(ok, MESSAGE_OK); + first = false; + } + // BUGBUG: The test is buggy: the client might not get any error when sending one-way messages. // All the messages that the client has sent before he gets first MESSAGE_BUSY error might get // serailized and written to the socket buffer, so the write queue gets drained and there are // no messages to timeout when periodic timeout check happens. - client.GotError.WaitI(); - } -} + client.GotError.WaitI(); + } +} diff --git a/library/cpp/messagebus/test/ut/starter_ut.cpp b/library/cpp/messagebus/test/ut/starter_ut.cpp index ebb628ab28..dd4d3aaa5e 100644 --- a/library/cpp/messagebus/test/ut/starter_ut.cpp +++ b/library/cpp/messagebus/test/ut/starter_ut.cpp @@ -1,140 +1,140 @@ #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/messagebus/test/helper/example_module.h> #include <library/cpp/messagebus/test/helper/object_count_check.h> #include <library/cpp/messagebus/test/helper/wait_for.h> - -using namespace NBus; -using namespace NBus::NTest; - + +using namespace NBus; +using namespace NBus::NTest; + Y_UNIT_TEST_SUITE(TBusStarterTest) { struct TStartJobTestModule: public TExampleModule { - using TBusModule::CreateDefaultStarter; - - TAtomic StartCount; - - TStartJobTestModule() - : StartCount(0) - { - } - + using TBusModule::CreateDefaultStarter; + + TAtomic StartCount; + + TStartJobTestModule() + : StartCount(0) + { + } + TJobHandler Start(TBusJob* job, TBusMessage* mess) override { Y_UNUSED(mess); - AtomicIncrement(StartCount); - job->Sleep(10); - return &TStartJobTestModule::End; - } - - TJobHandler End(TBusJob* job, TBusMessage* mess) { + AtomicIncrement(StartCount); + job->Sleep(10); + return &TStartJobTestModule::End; + } + + TJobHandler End(TBusJob* job, TBusMessage* mess) { Y_UNUSED(mess); - AtomicIncrement(StartCount); - job->Cancel(MESSAGE_UNKNOWN); + AtomicIncrement(StartCount); + job->Cancel(MESSAGE_UNKNOWN); return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(Test) { - TObjectCountCheck objectCountCheck; - - TBusMessageQueuePtr bus(CreateMessageQueue()); - - TStartJobTestModule module; - - //module.StartModule(); - module.CreatePrivateSessions(bus.Get()); - module.StartInput(); - - TBusSessionConfig config; - config.SendTimeout = 10; - - module.CreateDefaultStarter(*bus, config); - - UNIT_WAIT_FOR(AtomicGet(module.StartCount) >= 3); - - module.Shutdown(); - bus->Stop(); - } - + TObjectCountCheck objectCountCheck; + + TBusMessageQueuePtr bus(CreateMessageQueue()); + + TStartJobTestModule module; + + //module.StartModule(); + module.CreatePrivateSessions(bus.Get()); + module.StartInput(); + + TBusSessionConfig config; + config.SendTimeout = 10; + + module.CreateDefaultStarter(*bus, config); + + UNIT_WAIT_FOR(AtomicGet(module.StartCount) >= 3); + + module.Shutdown(); + bus->Stop(); + } + Y_UNIT_TEST(TestModuleStartJob) { - TObjectCountCheck objectCountCheck; - - TExampleProtocol proto; - - TStartJobTestModule module; - - TBusModuleConfig moduleConfig; - moduleConfig.Secret.SchedulePeriod = TDuration::MilliSeconds(10); - module.SetConfig(moduleConfig); - - module.StartModule(); - - module.StartJob(new TExampleRequest(&proto.RequestCount)); - - UNIT_WAIT_FOR(AtomicGet(module.StartCount) != 2); - - module.Shutdown(); - } - + TObjectCountCheck objectCountCheck; + + TExampleProtocol proto; + + TStartJobTestModule module; + + TBusModuleConfig moduleConfig; + moduleConfig.Secret.SchedulePeriod = TDuration::MilliSeconds(10); + module.SetConfig(moduleConfig); + + module.StartModule(); + + module.StartJob(new TExampleRequest(&proto.RequestCount)); + + UNIT_WAIT_FOR(AtomicGet(module.StartCount) != 2); + + module.Shutdown(); + } + struct TSleepModule: public TExampleServerModule { TSystemEvent MessageReceivedEvent; - + TJobHandler Start(TBusJob* job, TBusMessage* mess) override { Y_UNUSED(mess); - - MessageReceivedEvent.Signal(); - - job->Sleep(1000000000); - - return TJobHandler(&TSleepModule::Never); - } - - TJobHandler Never(TBusJob*, TBusMessage*) { + + MessageReceivedEvent.Signal(); + + job->Sleep(1000000000); + + return TJobHandler(&TSleepModule::Never); + } + + TJobHandler Never(TBusJob*, TBusMessage*) { Y_FAIL("happens"); - throw 1; - } - }; - + throw 1; + } + }; + Y_UNIT_TEST(StartJobDestroyDuringSleep) { - TObjectCountCheck objectCountCheck; - - TExampleProtocol proto; - - TSleepModule module; - - module.StartModule(); - - module.StartJob(new TExampleRequest(&proto.StartCount)); - - module.MessageReceivedEvent.WaitI(); - - module.Shutdown(); - } - + TObjectCountCheck objectCountCheck; + + TExampleProtocol proto; + + TSleepModule module; + + module.StartModule(); + + module.StartJob(new TExampleRequest(&proto.StartCount)); + + module.MessageReceivedEvent.WaitI(); + + module.Shutdown(); + } + struct TSendReplyModule: public TExampleServerModule { TSystemEvent MessageReceivedEvent; - + TJobHandler Start(TBusJob* job, TBusMessage* mess) override { Y_UNUSED(mess); - - job->SendReply(new TExampleResponse(&Proto.ResponseCount)); - - MessageReceivedEvent.Signal(); - + + job->SendReply(new TExampleResponse(&Proto.ResponseCount)); + + MessageReceivedEvent.Signal(); + return nullptr; - } - }; - + } + }; + Y_UNIT_TEST(AllowSendReplyInStarted) { - TObjectCountCheck objectCountCheck; - - TExampleProtocol proto; - - TSendReplyModule module; - module.StartModule(); - module.StartJob(new TExampleRequest(&proto.StartCount)); - - module.MessageReceivedEvent.WaitI(); - - module.Shutdown(); - } -} + TObjectCountCheck objectCountCheck; + + TExampleProtocol proto; + + TSendReplyModule module; + module.StartModule(); + module.StartJob(new TExampleRequest(&proto.StartCount)); + + module.MessageReceivedEvent.WaitI(); + + module.Shutdown(); + } +} diff --git a/library/cpp/messagebus/test/ut/sync_client_ut.cpp b/library/cpp/messagebus/test/ut/sync_client_ut.cpp index 848a9d3457..400128193f 100644 --- a/library/cpp/messagebus/test/ut/sync_client_ut.cpp +++ b/library/cpp/messagebus/test/ut/sync_client_ut.cpp @@ -4,7 +4,7 @@ namespace NBus { namespace NTest { using namespace std; - + //////////////////////////////////////////////////////////////////// /// \brief Client for sending synchronous message to local server struct TSyncClient { @@ -13,7 +13,7 @@ namespace NBus { TExampleProtocol Proto; TBusMessageQueuePtr Bus; TBusSyncClientSessionPtr Session; - + int NumReplies; int NumMessages; @@ -53,7 +53,7 @@ namespace NBus { Y_UNIT_TEST_SUITE(SyncClientTest) { Y_UNIT_TEST(TestSync) { TObjectCountCheck objectCountCheck; - + TExampleServer server; TSyncClient client(server.GetActualListenAddr()); client.Work(); @@ -65,5 +65,5 @@ namespace NBus { } } - } -} + } +} diff --git a/library/cpp/messagebus/test/ut/ya.make b/library/cpp/messagebus/test/ut/ya.make index 5af102e0ba..fe1b4961d6 100644 --- a/library/cpp/messagebus/test/ut/ya.make +++ b/library/cpp/messagebus/test/ut/ya.make @@ -1,7 +1,7 @@ OWNER(g:messagebus) - + UNITTEST_FOR(library/cpp/messagebus) - + TIMEOUT(1200) SIZE(LARGE) diff --git a/library/cpp/messagebus/test/ya.make b/library/cpp/messagebus/test/ya.make index 1c1f8bbd9c..0dc4bd4720 100644 --- a/library/cpp/messagebus/test/ya.make +++ b/library/cpp/messagebus/test/ya.make @@ -1,5 +1,5 @@ OWNER(g:messagebus) - + RECURSE( example perftest diff --git a/library/cpp/messagebus/text_utils.h b/library/cpp/messagebus/text_utils.h index 75489d1b07..c2dcad834c 100644 --- a/library/cpp/messagebus/text_utils.h +++ b/library/cpp/messagebus/text_utils.h @@ -1,3 +1,3 @@ -#pragma once - +#pragma once + #include <library/cpp/string_utils/indent_text/indent_text.h> diff --git a/library/cpp/messagebus/thread_extra.h b/library/cpp/messagebus/thread_extra.h index 9b19d516fa..2c79741e88 100644 --- a/library/cpp/messagebus/thread_extra.h +++ b/library/cpp/messagebus/thread_extra.h @@ -1,3 +1,3 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/actor/thread_extra.h> diff --git a/library/cpp/messagebus/use_after_free_checker.cpp b/library/cpp/messagebus/use_after_free_checker.cpp index f6586f8e7f..4904e7c614 100644 --- a/library/cpp/messagebus/use_after_free_checker.cpp +++ b/library/cpp/messagebus/use_after_free_checker.cpp @@ -1,22 +1,22 @@ #include "use_after_free_checker.h" -#include <util/system/yassert.h> - -namespace { +#include <util/system/yassert.h> + +namespace { const ui64 VALID = (ui64)0xAABBCCDDEEFF0011LL; const ui64 INVALID = (ui64)0x1122334455667788LL; -} - -TUseAfterFreeChecker::TUseAfterFreeChecker() - : Magic(VALID) -{ -} - +} + +TUseAfterFreeChecker::TUseAfterFreeChecker() + : Magic(VALID) +{ +} + TUseAfterFreeChecker::~TUseAfterFreeChecker() { Y_VERIFY(Magic == VALID, "Corrupted"); - Magic = INVALID; -} - -void TUseAfterFreeChecker::CheckNotFreed() const { + Magic = INVALID; +} + +void TUseAfterFreeChecker::CheckNotFreed() const { Y_VERIFY(Magic == VALID, "Freed or corrupted"); -} +} diff --git a/library/cpp/messagebus/use_after_free_checker.h b/library/cpp/messagebus/use_after_free_checker.h index 077abc4d92..590b076156 100644 --- a/library/cpp/messagebus/use_after_free_checker.h +++ b/library/cpp/messagebus/use_after_free_checker.h @@ -1,31 +1,31 @@ -#pragma once - -#include <util/system/platform.h> +#pragma once + +#include <util/system/platform.h> #include <util/system/types.h> - -class TUseAfterFreeChecker { -private: - ui64 Magic; -public: - TUseAfterFreeChecker(); - ~TUseAfterFreeChecker(); - void CheckNotFreed() const; -}; - -// check twice: in constructor and in destructor -class TUseAfterFreeCheckerGuard { -private: - const TUseAfterFreeChecker& Check; +class TUseAfterFreeChecker { +private: + ui64 Magic; + +public: + TUseAfterFreeChecker(); + ~TUseAfterFreeChecker(); + void CheckNotFreed() const; +}; + +// check twice: in constructor and in destructor +class TUseAfterFreeCheckerGuard { +private: + const TUseAfterFreeChecker& Check; + +public: + TUseAfterFreeCheckerGuard(const TUseAfterFreeChecker& check) + : Check(check) + { + Check.CheckNotFreed(); + } -public: - TUseAfterFreeCheckerGuard(const TUseAfterFreeChecker& check) - : Check(check) - { - Check.CheckNotFreed(); - } - - ~TUseAfterFreeCheckerGuard() { - Check.CheckNotFreed(); - } -}; + ~TUseAfterFreeCheckerGuard() { + Check.CheckNotFreed(); + } +}; diff --git a/library/cpp/messagebus/use_count_checker.cpp b/library/cpp/messagebus/use_count_checker.cpp index 5b253aca6f..c6243ea21f 100644 --- a/library/cpp/messagebus/use_count_checker.cpp +++ b/library/cpp/messagebus/use_count_checker.cpp @@ -1,53 +1,53 @@ #include "use_count_checker.h" #include <util/generic/utility.h> -#include <util/system/yassert.h> - -TUseCountChecker::TUseCountChecker() { -} - -TUseCountChecker::~TUseCountChecker() { - TAtomicBase count = Counter.Val(); +#include <util/system/yassert.h> + +TUseCountChecker::TUseCountChecker() { +} + +TUseCountChecker::~TUseCountChecker() { + TAtomicBase count = Counter.Val(); Y_VERIFY(count == 0, "must not release when count is not zero: %ld", (long)count); -} - -void TUseCountChecker::Inc() { - Counter.Inc(); -} - -void TUseCountChecker::Dec() { - Counter.Dec(); -} - -TUseCountHolder::TUseCountHolder() +} + +void TUseCountChecker::Inc() { + Counter.Inc(); +} + +void TUseCountChecker::Dec() { + Counter.Dec(); +} + +TUseCountHolder::TUseCountHolder() : CurrentChecker(nullptr) { } - -TUseCountHolder::TUseCountHolder(TUseCountChecker* currentChecker) - : CurrentChecker(currentChecker) -{ - if (!!CurrentChecker) { - CurrentChecker->Inc(); - } -} - + +TUseCountHolder::TUseCountHolder(TUseCountChecker* currentChecker) + : CurrentChecker(currentChecker) +{ + if (!!CurrentChecker) { + CurrentChecker->Inc(); + } +} + TUseCountHolder::~TUseCountHolder() { - if (!!CurrentChecker) { - CurrentChecker->Dec(); - } -} - -TUseCountHolder& TUseCountHolder::operator=(TUseCountHolder that) { - Swap(that); - return *this; -} - -void TUseCountHolder::Swap(TUseCountHolder& that) { - DoSwap(CurrentChecker, that.CurrentChecker); -} - -void TUseCountHolder::Reset() { - TUseCountHolder tmp; - Swap(tmp); -} + if (!!CurrentChecker) { + CurrentChecker->Dec(); + } +} + +TUseCountHolder& TUseCountHolder::operator=(TUseCountHolder that) { + Swap(that); + return *this; +} + +void TUseCountHolder::Swap(TUseCountHolder& that) { + DoSwap(CurrentChecker, that.CurrentChecker); +} + +void TUseCountHolder::Reset() { + TUseCountHolder tmp; + Swap(tmp); +} diff --git a/library/cpp/messagebus/use_count_checker.h b/library/cpp/messagebus/use_count_checker.h index 9437bace98..70bef6fa8a 100644 --- a/library/cpp/messagebus/use_count_checker.h +++ b/library/cpp/messagebus/use_count_checker.h @@ -1,27 +1,27 @@ -#pragma once - -#include <util/generic/refcount.h> - -class TUseCountChecker { -private: - TAtomicCounter Counter; +#pragma once -public: - TUseCountChecker(); - ~TUseCountChecker(); - void Inc(); - void Dec(); -}; - -class TUseCountHolder { -private: - TUseCountChecker* CurrentChecker; +#include <util/generic/refcount.h> -public: - TUseCountHolder(); - explicit TUseCountHolder(TUseCountChecker* currentChecker); - TUseCountHolder& operator=(TUseCountHolder that); - ~TUseCountHolder(); - void Swap(TUseCountHolder&); - void Reset(); -}; +class TUseCountChecker { +private: + TAtomicCounter Counter; + +public: + TUseCountChecker(); + ~TUseCountChecker(); + void Inc(); + void Dec(); +}; + +class TUseCountHolder { +private: + TUseCountChecker* CurrentChecker; + +public: + TUseCountHolder(); + explicit TUseCountHolder(TUseCountChecker* currentChecker); + TUseCountHolder& operator=(TUseCountHolder that); + ~TUseCountHolder(); + void Swap(TUseCountHolder&); + void Reset(); +}; diff --git a/library/cpp/messagebus/vector_swaps.h b/library/cpp/messagebus/vector_swaps.h index 510ee39ef9..b920bcf03e 100644 --- a/library/cpp/messagebus/vector_swaps.h +++ b/library/cpp/messagebus/vector_swaps.h @@ -1,171 +1,171 @@ -#pragma once - +#pragma once + #include <util/generic/array_ref.h> -#include <util/generic/noncopyable.h> +#include <util/generic/noncopyable.h> #include <util/generic/utility.h> -#include <util/system/yassert.h> - +#include <util/system/yassert.h> + #include <stdlib.h> template <typename T, class A = std::allocator<T>> class TVectorSwaps : TNonCopyable { -private: - T* Start; - T* Finish; - T* EndOfStorage; - - void StateCheck() { +private: + T* Start; + T* Finish; + T* EndOfStorage; + + void StateCheck() { Y_ASSERT(Start <= Finish); Y_ASSERT(Finish <= EndOfStorage); - } - -public: - typedef T* iterator; - typedef const T* const_iterator; - + } + +public: + typedef T* iterator; + typedef const T* const_iterator; + typedef std::reverse_iterator<iterator> reverse_iterator; typedef std::reverse_iterator<const_iterator> const_reverse_iterator; - + TVectorSwaps() : Start() , Finish() , EndOfStorage() { } - - ~TVectorSwaps() { - for (size_t i = 0; i < size(); ++i) { - Start[i].~T(); - } - free(Start); - } - + + ~TVectorSwaps() { + for (size_t i = 0; i < size(); ++i) { + Start[i].~T(); + } + free(Start); + } + operator TArrayRef<const T>() const { return MakeArrayRef(data(), size()); - } - + } + operator TArrayRef<T>() { return MakeArrayRef(data(), size()); - } - - size_t capacity() const { - return EndOfStorage - Start; - } - - size_t size() const { - return Finish - Start; - } - - bool empty() const { - return size() == 0; - } - - T* data() { - return Start; - } - - const T* data() const { - return Start; - } - - T& operator[](size_t index) { + } + + size_t capacity() const { + return EndOfStorage - Start; + } + + size_t size() const { + return Finish - Start; + } + + bool empty() const { + return size() == 0; + } + + T* data() { + return Start; + } + + const T* data() const { + return Start; + } + + T& operator[](size_t index) { Y_ASSERT(index < size()); - return Start[index]; - } - - const T& operator[](size_t index) const { + return Start[index]; + } + + const T& operator[](size_t index) const { Y_ASSERT(index < size()); - return Start[index]; - } - - iterator begin() { - return Start; - } - - iterator end() { - return Finish; - } - - const_iterator begin() const { - return Start; - } - - const_iterator end() const { - return Finish; - } - + return Start[index]; + } + + iterator begin() { + return Start; + } + + iterator end() { + return Finish; + } + + const_iterator begin() const { + return Start; + } + + const_iterator end() const { + return Finish; + } + reverse_iterator rbegin() { return reverse_iterator(end()); } reverse_iterator rend() { return reverse_iterator(begin()); } - + const_reverse_iterator rbegin() const { return reverse_iterator(end()); } const_reverse_iterator rend() const { return reverse_iterator(begin()); } - - void swap(TVectorSwaps<T>& that) { - DoSwap(Start, that.Start); - DoSwap(Finish, that.Finish); - DoSwap(EndOfStorage, that.EndOfStorage); - } - - void reserve(size_t n) { - if (n <= capacity()) { - return; - } - - size_t newCapacity = FastClp2(n); - TVectorSwaps<T> tmp; + + void swap(TVectorSwaps<T>& that) { + DoSwap(Start, that.Start); + DoSwap(Finish, that.Finish); + DoSwap(EndOfStorage, that.EndOfStorage); + } + + void reserve(size_t n) { + if (n <= capacity()) { + return; + } + + size_t newCapacity = FastClp2(n); + TVectorSwaps<T> tmp; tmp.Start = (T*)malloc(sizeof(T) * newCapacity); Y_VERIFY(!!tmp.Start); - - tmp.EndOfStorage = tmp.Start + newCapacity; - - for (size_t i = 0; i < size(); ++i) { - // TODO: catch exceptions - new (tmp.Start + i) T(); - DoSwap(Start[i], tmp.Start[i]); - } - - tmp.Finish = tmp.Start + size(); - - swap(tmp); - - StateCheck(); - } - - void clear() { - TVectorSwaps<T> tmp; - swap(tmp); - } - - template <class TIterator> - void insert(iterator pos, TIterator b, TIterator e) { + + tmp.EndOfStorage = tmp.Start + newCapacity; + + for (size_t i = 0; i < size(); ++i) { + // TODO: catch exceptions + new (tmp.Start + i) T(); + DoSwap(Start[i], tmp.Start[i]); + } + + tmp.Finish = tmp.Start + size(); + + swap(tmp); + + StateCheck(); + } + + void clear() { + TVectorSwaps<T> tmp; + swap(tmp); + } + + template <class TIterator> + void insert(iterator pos, TIterator b, TIterator e) { Y_VERIFY(pos == end(), "TODO: only insert at the end is implemented"); - - size_t count = e - b; - - reserve(size() + count); - - TIterator next = b; - - for (size_t i = 0; i < count; ++i) { - new (Start + size() + i) T(); - DoSwap(Start[size() + i], *next); - ++next; - } - - Finish += count; - - StateCheck(); - } - - void push_back(T& elem) { + + size_t count = e - b; + + reserve(size() + count); + + TIterator next = b; + + for (size_t i = 0; i < count; ++i) { + new (Start + size() + i) T(); + DoSwap(Start[size() + i], *next); + ++next; + } + + Finish += count; + + StateCheck(); + } + + void push_back(T& elem) { insert(end(), &elem, &elem + 1); - } -}; + } +}; diff --git a/library/cpp/messagebus/vector_swaps_ut.cpp b/library/cpp/messagebus/vector_swaps_ut.cpp index d30544092d..693cc6857b 100644 --- a/library/cpp/messagebus/vector_swaps_ut.cpp +++ b/library/cpp/messagebus/vector_swaps_ut.cpp @@ -1,17 +1,17 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "vector_swaps.h" - + +#include "vector_swaps.h" + Y_UNIT_TEST_SUITE(TVectorSwapsTest) { Y_UNIT_TEST(Simple) { TVectorSwaps<THolder<unsigned>> v; - for (unsigned i = 0; i < 100; ++i) { - THolder<unsigned> tmp(new unsigned(i)); - v.push_back(tmp); - } - - for (unsigned i = 0; i < 100; ++i) { - UNIT_ASSERT_VALUES_EQUAL(i, *v[i]); - } - } -} + for (unsigned i = 0; i < 100; ++i) { + THolder<unsigned> tmp(new unsigned(i)); + v.push_back(tmp); + } + + for (unsigned i = 0; i < 100; ++i) { + UNIT_ASSERT_VALUES_EQUAL(i, *v[i]); + } + } +} diff --git a/library/cpp/messagebus/www/concat_strings.h b/library/cpp/messagebus/www/concat_strings.h index bfaeab2c76..7b730564eb 100644 --- a/library/cpp/messagebus/www/concat_strings.h +++ b/library/cpp/messagebus/www/concat_strings.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include <util/generic/string.h> -#include <util/stream/str.h> - +#include <util/stream/str.h> + // ATTN: not equivalent to TString::Join - cat concat anything "outputable" to stream, not only TString convertable types. - + inline void DoConcatStrings(TStringStream&) { -} - +} + template <class T, class... R> inline void DoConcatStrings(TStringStream& ss, const T& t, const R&... r) { ss << t; DoConcatStrings(ss, r...); -} - +} + template <class... R> inline TString ConcatStrings(const R&... r) { - TStringStream ss; + TStringStream ss; DoConcatStrings(ss, r...); - return ss.Str(); -} + return ss.Str(); +} diff --git a/library/cpp/messagebus/www/html_output.cpp b/library/cpp/messagebus/www/html_output.cpp index 9a35c5ca53..10ea2e163b 100644 --- a/library/cpp/messagebus/www/html_output.cpp +++ b/library/cpp/messagebus/www/html_output.cpp @@ -1,4 +1,4 @@ -#include "html_output.h" - +#include "html_output.h" + Y_POD_THREAD(IOutputStream*) HtmlOutputStreamPtr; diff --git a/library/cpp/messagebus/www/html_output.h b/library/cpp/messagebus/www/html_output.h index ff1c5da17b..27e77adefa 100644 --- a/library/cpp/messagebus/www/html_output.h +++ b/library/cpp/messagebus/www/html_output.h @@ -1,37 +1,37 @@ -#pragma once - +#pragma once + #include "concat_strings.h" #include <util/generic/string.h> -#include <util/stream/output.h> +#include <util/stream/output.h> #include <library/cpp/html/pcdata/pcdata.h> -#include <util/system/tls.h> - +#include <util/system/tls.h> + extern Y_POD_THREAD(IOutputStream*) HtmlOutputStreamPtr; - + static IOutputStream& HtmlOutputStream() { Y_VERIFY(!!HtmlOutputStreamPtr); - return *HtmlOutputStreamPtr; -} - -struct THtmlOutputStreamPushPop { + return *HtmlOutputStreamPtr; +} + +struct THtmlOutputStreamPushPop { IOutputStream* const Prev; - + THtmlOutputStreamPushPop(IOutputStream* outputStream) - : Prev(HtmlOutputStreamPtr) - { - HtmlOutputStreamPtr = outputStream; - } - - ~THtmlOutputStreamPushPop() { - HtmlOutputStreamPtr = Prev; - } -}; - -struct TChars { + : Prev(HtmlOutputStreamPtr) + { + HtmlOutputStreamPtr = outputStream; + } + + ~THtmlOutputStreamPushPop() { + HtmlOutputStreamPtr = Prev; + } +}; + +struct TChars { TString Text; - bool NeedEscape; - + bool NeedEscape; + TChars(TStringBuf text) : Text(text) , NeedEscape(true) @@ -52,273 +52,273 @@ struct TChars { , NeedEscape(escape) { } - + TString Escape() { - if (NeedEscape) { - return EncodeHtmlPcdata(Text); - } else { - return Text; - } - } -}; - -struct TAttr { + if (NeedEscape) { + return EncodeHtmlPcdata(Text); + } else { + return Text; + } + } +}; + +struct TAttr { TString Name; TString Value; - - TAttr(TStringBuf name, TStringBuf value) + + TAttr(TStringBuf name, TStringBuf value) : Name(name) , Value(value) { } - + TAttr() { } - - bool operator!() const { - return !Name; - } -}; - -static inline void Doctype() { - HtmlOutputStream() << "<!doctype html>\n"; -} - -static inline void Nl() { - HtmlOutputStream() << "\n"; -} - -static inline void Sp() { - HtmlOutputStream() << " "; -} - -static inline void Text(TStringBuf text) { - HtmlOutputStream() << EncodeHtmlPcdata(text); -} - -static inline void Line(TStringBuf text) { - Text(text); - Nl(); -} - -static inline void WriteAttr(TAttr a) { - if (!!a) { - HtmlOutputStream() << " " << a.Name << "='" << EncodeHtmlPcdata(a.Value) << "'"; - } -} - -static inline void Open(TStringBuf tag, TAttr a1 = TAttr(), TAttr a2 = TAttr(), TAttr a3 = TAttr(), TAttr a4 = TAttr()) { - HtmlOutputStream() << "<" << tag; - WriteAttr(a1); - WriteAttr(a2); - WriteAttr(a3); - WriteAttr(a4); - HtmlOutputStream() << ">"; -} - -static inline void Open(TStringBuf tag, TStringBuf cssClass, TStringBuf id = "") { - Open(tag, TAttr("class", cssClass), !!id ? TAttr("id", id) : TAttr()); -} - -static inline void OpenBlock(TStringBuf tag, TStringBuf cssClass = "") { - Open(tag, cssClass); - Nl(); -} - -static inline void Close(TStringBuf tag) { - HtmlOutputStream() << "</" << tag << ">\n"; -} - -static inline void CloseBlock(TStringBuf tag) { - Close(tag); - Nl(); -} - -static inline void TagWithContent(TStringBuf tag, TChars content) { - HtmlOutputStream() << "<" << tag << ">" << content.Escape() << "</" << tag << ">"; -} - -static inline void BlockTagWithContent(TStringBuf tag, TStringBuf content) { - TagWithContent(tag, content); - Nl(); -} - -static inline void TagWithClass(TStringBuf tag, TStringBuf cssClass) { - Open(tag, cssClass); - Close(tag); -} - -static inline void Hn(unsigned n, TStringBuf title) { - BlockTagWithContent(ConcatStrings("h", n), title); -} - -static inline void Small(TStringBuf text) { - TagWithContent("small", text); -} - -static inline void HnWithSmall(unsigned n, TStringBuf title, TStringBuf small) { + + bool operator!() const { + return !Name; + } +}; + +static inline void Doctype() { + HtmlOutputStream() << "<!doctype html>\n"; +} + +static inline void Nl() { + HtmlOutputStream() << "\n"; +} + +static inline void Sp() { + HtmlOutputStream() << " "; +} + +static inline void Text(TStringBuf text) { + HtmlOutputStream() << EncodeHtmlPcdata(text); +} + +static inline void Line(TStringBuf text) { + Text(text); + Nl(); +} + +static inline void WriteAttr(TAttr a) { + if (!!a) { + HtmlOutputStream() << " " << a.Name << "='" << EncodeHtmlPcdata(a.Value) << "'"; + } +} + +static inline void Open(TStringBuf tag, TAttr a1 = TAttr(), TAttr a2 = TAttr(), TAttr a3 = TAttr(), TAttr a4 = TAttr()) { + HtmlOutputStream() << "<" << tag; + WriteAttr(a1); + WriteAttr(a2); + WriteAttr(a3); + WriteAttr(a4); + HtmlOutputStream() << ">"; +} + +static inline void Open(TStringBuf tag, TStringBuf cssClass, TStringBuf id = "") { + Open(tag, TAttr("class", cssClass), !!id ? TAttr("id", id) : TAttr()); +} + +static inline void OpenBlock(TStringBuf tag, TStringBuf cssClass = "") { + Open(tag, cssClass); + Nl(); +} + +static inline void Close(TStringBuf tag) { + HtmlOutputStream() << "</" << tag << ">\n"; +} + +static inline void CloseBlock(TStringBuf tag) { + Close(tag); + Nl(); +} + +static inline void TagWithContent(TStringBuf tag, TChars content) { + HtmlOutputStream() << "<" << tag << ">" << content.Escape() << "</" << tag << ">"; +} + +static inline void BlockTagWithContent(TStringBuf tag, TStringBuf content) { + TagWithContent(tag, content); + Nl(); +} + +static inline void TagWithClass(TStringBuf tag, TStringBuf cssClass) { + Open(tag, cssClass); + Close(tag); +} + +static inline void Hn(unsigned n, TStringBuf title) { + BlockTagWithContent(ConcatStrings("h", n), title); +} + +static inline void Small(TStringBuf text) { + TagWithContent("small", text); +} + +static inline void HnWithSmall(unsigned n, TStringBuf title, TStringBuf small) { TString tagName = ConcatStrings("h", n); - Open(tagName); - HtmlOutputStream() << title; - Sp(); - Small(small); - Close(tagName); -} - -static inline void H1(TStringBuf title) { - Hn(1, title); -} - -static inline void H2(TStringBuf title) { - Hn(2, title); -} - -static inline void H3(TStringBuf title) { - Hn(3, title); -} - -static inline void H4(TStringBuf title) { - Hn(4, title); -} - -static inline void H5(TStringBuf title) { - Hn(5, title); -} - -static inline void H6(TStringBuf title) { - Hn(6, title); -} - -static inline void Pre(TStringBuf content) { - HtmlOutputStream() << "<pre>" << EncodeHtmlPcdata(content) << "</pre>\n"; -} - -static inline void Li(TStringBuf content) { - BlockTagWithContent("li", content); -} - -static inline void LiWithClass(TStringBuf cssClass, TStringBuf content) { - Open("li", cssClass); - Text(content); - Close("li"); -} - -static inline void OpenA(TStringBuf href) { - Open("a", TAttr("href", href)); -} - -static inline void A(TStringBuf href, TStringBuf text) { - OpenA(href); - Text(text); - Close("a"); -} - -static inline void Td(TStringBuf content) { - TagWithContent("td", content); -} - -static inline void Th(TStringBuf content, TStringBuf cssClass = "") { - OpenBlock("th", cssClass); - Text(content); - CloseBlock("th"); -} - -static inline void DivWithClassAndContent(TStringBuf cssClass, TStringBuf content) { - Open("div", cssClass); - Text(content); - Close("div"); -} - -static inline void BootstrapError(TStringBuf text) { - DivWithClassAndContent("alert alert-danger", text); -} - -static inline void BootstrapInfo(TStringBuf text) { - DivWithClassAndContent("alert alert-info", text); -} - -static inline void ScriptHref(TStringBuf href) { - Open("script", + Open(tagName); + HtmlOutputStream() << title; + Sp(); + Small(small); + Close(tagName); +} + +static inline void H1(TStringBuf title) { + Hn(1, title); +} + +static inline void H2(TStringBuf title) { + Hn(2, title); +} + +static inline void H3(TStringBuf title) { + Hn(3, title); +} + +static inline void H4(TStringBuf title) { + Hn(4, title); +} + +static inline void H5(TStringBuf title) { + Hn(5, title); +} + +static inline void H6(TStringBuf title) { + Hn(6, title); +} + +static inline void Pre(TStringBuf content) { + HtmlOutputStream() << "<pre>" << EncodeHtmlPcdata(content) << "</pre>\n"; +} + +static inline void Li(TStringBuf content) { + BlockTagWithContent("li", content); +} + +static inline void LiWithClass(TStringBuf cssClass, TStringBuf content) { + Open("li", cssClass); + Text(content); + Close("li"); +} + +static inline void OpenA(TStringBuf href) { + Open("a", TAttr("href", href)); +} + +static inline void A(TStringBuf href, TStringBuf text) { + OpenA(href); + Text(text); + Close("a"); +} + +static inline void Td(TStringBuf content) { + TagWithContent("td", content); +} + +static inline void Th(TStringBuf content, TStringBuf cssClass = "") { + OpenBlock("th", cssClass); + Text(content); + CloseBlock("th"); +} + +static inline void DivWithClassAndContent(TStringBuf cssClass, TStringBuf content) { + Open("div", cssClass); + Text(content); + Close("div"); +} + +static inline void BootstrapError(TStringBuf text) { + DivWithClassAndContent("alert alert-danger", text); +} + +static inline void BootstrapInfo(TStringBuf text) { + DivWithClassAndContent("alert alert-info", text); +} + +static inline void ScriptHref(TStringBuf href) { + Open("script", TAttr("language", "javascript"), TAttr("type", "text/javascript"), TAttr("src", href)); - Close("script"); - Nl(); -} - -static inline void LinkStylesheet(TStringBuf href) { - Open("link", TAttr("rel", "stylesheet"), TAttr("href", href)); - Close("link"); - Nl(); -} - -static inline void LinkFavicon(TStringBuf href) { - Open("link", TAttr("rel", "shortcut icon"), TAttr("href", href)); - Close("link"); - Nl(); -} - -static inline void Title(TChars title) { - TagWithContent("title", title); - Nl(); -} - -static inline void Code(TStringBuf content) { - TagWithContent("code", content); -} - -struct TTagGuard { + Close("script"); + Nl(); +} + +static inline void LinkStylesheet(TStringBuf href) { + Open("link", TAttr("rel", "stylesheet"), TAttr("href", href)); + Close("link"); + Nl(); +} + +static inline void LinkFavicon(TStringBuf href) { + Open("link", TAttr("rel", "shortcut icon"), TAttr("href", href)); + Close("link"); + Nl(); +} + +static inline void Title(TChars title) { + TagWithContent("title", title); + Nl(); +} + +static inline void Code(TStringBuf content) { + TagWithContent("code", content); +} + +struct TTagGuard { const TString TagName; - - TTagGuard(TStringBuf tagName, TStringBuf cssClass, TStringBuf id = "") - : TagName(tagName) - { - Open(TagName, cssClass, id); - } - - TTagGuard(TStringBuf tagName, TAttr a1 = TAttr(), TAttr a2 = TAttr(), TAttr a3 = TAttr(), TAttr a4 = TAttr()) - : TagName(tagName) - { - Open(tagName, a1, a2, a3, a4); - } - - ~TTagGuard() { - Close(TagName); - } -}; - -struct TDivGuard: public TTagGuard { - TDivGuard(TStringBuf cssClass, TStringBuf id = "") - : TTagGuard("div", cssClass, id) + + TTagGuard(TStringBuf tagName, TStringBuf cssClass, TStringBuf id = "") + : TagName(tagName) { + Open(TagName, cssClass, id); } - - TDivGuard(TAttr a1 = TAttr(), TAttr a2 = TAttr(), TAttr a3 = TAttr()) - : TTagGuard("div", a1, a2, a3) + + TTagGuard(TStringBuf tagName, TAttr a1 = TAttr(), TAttr a2 = TAttr(), TAttr a3 = TAttr(), TAttr a4 = TAttr()) + : TagName(tagName) + { + Open(tagName, a1, a2, a3, a4); + } + + ~TTagGuard() { + Close(TagName); + } +}; + +struct TDivGuard: public TTagGuard { + TDivGuard(TStringBuf cssClass, TStringBuf id = "") + : TTagGuard("div", cssClass, id) { } -}; - -struct TAGuard { + + TDivGuard(TAttr a1 = TAttr(), TAttr a2 = TAttr(), TAttr a3 = TAttr()) + : TTagGuard("div", a1, a2, a3) + { + } +}; + +struct TAGuard { TAGuard(TStringBuf href) { - OpenA(href); - } - - ~TAGuard() { - Close("a"); - } -}; - -struct TScriptFunctionGuard { - TTagGuard Script; - - TScriptFunctionGuard() - : Script("script") - { - Line("$(function() {"); - } - + OpenA(href); + } + + ~TAGuard() { + Close("a"); + } +}; + +struct TScriptFunctionGuard { + TTagGuard Script; + + TScriptFunctionGuard() + : Script("script") + { + Line("$(function() {"); + } + ~TScriptFunctionGuard() { - Line("});"); - } -}; + Line("});"); + } +}; diff --git a/library/cpp/messagebus/www/messagebus.js b/library/cpp/messagebus/www/messagebus.js index 2781f47df3..e30508b879 100644 --- a/library/cpp/messagebus/www/messagebus.js +++ b/library/cpp/messagebus/www/messagebus.js @@ -1,48 +1,48 @@ -function logTransform(v) { - return Math.log(v + 1); -} - -function plotHist(where, hist) { +function logTransform(v) { + return Math.log(v + 1); +} + +function plotHist(where, hist) { var max = hist.map(function(x) {return x[1]}).reduce(function(x, y) {return Math.max(x, y)}); - - var ticks = []; - for (var t = 1; ; t *= 10) { - if (t > max) { - break; - } - ticks.push(t); - } - - $.plot(where, [hist], - { - data: hist, - series: { - bars: { - show: true, - barWidth: 0.9 - } - }, - xaxis: { - mode: 'categories', - tickLength: 0 - }, - yaxis: { - ticks: ticks, - transform: logTransform - } - } - ); -} - -function plotQueueSize(where, data, ticks) { - $.plot(where, [data], - { - xaxis: { - ticks: ticks, - }, - yaxis: { - //transform: logTransform - } - } - ); -} + + var ticks = []; + for (var t = 1; ; t *= 10) { + if (t > max) { + break; + } + ticks.push(t); + } + + $.plot(where, [hist], + { + data: hist, + series: { + bars: { + show: true, + barWidth: 0.9 + } + }, + xaxis: { + mode: 'categories', + tickLength: 0 + }, + yaxis: { + ticks: ticks, + transform: logTransform + } + } + ); +} + +function plotQueueSize(where, data, ticks) { + $.plot(where, [data], + { + xaxis: { + ticks: ticks, + }, + yaxis: { + //transform: logTransform + } + } + ); +} diff --git a/library/cpp/messagebus/www/www.cpp b/library/cpp/messagebus/www/www.cpp index 90c50aacfc..62ec241d85 100644 --- a/library/cpp/messagebus/www/www.cpp +++ b/library/cpp/messagebus/www/www.cpp @@ -1,8 +1,8 @@ #include "www.h" - + #include "concat_strings.h" #include "html_output.h" - + #include <library/cpp/messagebus/remote_connection_status.h> #include <library/cpp/monlib/deprecated/json/writer.h> @@ -12,243 +12,243 @@ #include <library/cpp/http/server/http.h> #include <library/cpp/json/writer/json.h> #include <library/cpp/uri/http_url.h> - + #include <util/string/cast.h> #include <util/string/printf.h> #include <util/system/mutex.h> - + #include <utility> - -using namespace NBus; -using namespace NBus::NPrivate; -using namespace NActor; -using namespace NActor::NPrivate; - -static const char HTTP_OK_JS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/javascript\r\nConnection: Close\r\n\r\n"; -static const char HTTP_OK_JSON[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/json; charset=utf-8\r\nConnection: Close\r\n\r\n"; -static const char HTTP_OK_PNG[] = "HTTP/1.1 200 Ok\r\nContent-Type: image/png\r\nConnection: Close\r\n\r\n"; -static const char HTTP_OK_BIN[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/octet-stream\r\nConnection: Close\r\n\r\n"; -static const char HTTP_OK_HTML[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/html; charset=utf-8\r\nConnection: Close\r\n\r\n"; - -namespace { - typedef TIntrusivePtr<TBusModuleInternal> TBusModuleInternalPtr; - - template <typename TValuePtr> - struct TNamedValues { + +using namespace NBus; +using namespace NBus::NPrivate; +using namespace NActor; +using namespace NActor::NPrivate; + +static const char HTTP_OK_JS[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/javascript\r\nConnection: Close\r\n\r\n"; +static const char HTTP_OK_JSON[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/json; charset=utf-8\r\nConnection: Close\r\n\r\n"; +static const char HTTP_OK_PNG[] = "HTTP/1.1 200 Ok\r\nContent-Type: image/png\r\nConnection: Close\r\n\r\n"; +static const char HTTP_OK_BIN[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/octet-stream\r\nConnection: Close\r\n\r\n"; +static const char HTTP_OK_HTML[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/html; charset=utf-8\r\nConnection: Close\r\n\r\n"; + +namespace { + typedef TIntrusivePtr<TBusModuleInternal> TBusModuleInternalPtr; + + template <typename TValuePtr> + struct TNamedValues { TVector<std::pair<TString, TValuePtr>> Entries; - - TValuePtr FindByName(TStringBuf name) { + + TValuePtr FindByName(TStringBuf name) { Y_VERIFY(!!name); - - for (unsigned i = 0; i < Entries.size(); ++i) { - if (Entries[i].first == name) { - return Entries[i].second; - } - } - return TValuePtr(); - } - + + for (unsigned i = 0; i < Entries.size(); ++i) { + if (Entries[i].first == name) { + return Entries[i].second; + } + } + return TValuePtr(); + } + TString FindNameByPtr(TValuePtr value) { Y_VERIFY(!!value); - - for (unsigned i = 0; i < Entries.size(); ++i) { - if (Entries[i].second.Get() == value.Get()) { - return Entries[i].first; - } - } - + + for (unsigned i = 0; i < Entries.size(); ++i) { + if (Entries[i].second.Get() == value.Get()) { + return Entries[i].first; + } + } + Y_FAIL("unregistered"); - } - - void Add(TValuePtr p) { + } + + void Add(TValuePtr p) { Y_VERIFY(!!p); - - // Do not add twice - for (unsigned i = 0; i < Entries.size(); ++i) { - if (Entries[i].second.Get() == p.Get()) { - return; - } - } - - if (!!p->GetNameInternal()) { - TValuePtr current = FindByName(p->GetNameInternal()); - - if (!current) { + + // Do not add twice + for (unsigned i = 0; i < Entries.size(); ++i) { + if (Entries[i].second.Get() == p.Get()) { + return; + } + } + + if (!!p->GetNameInternal()) { + TValuePtr current = FindByName(p->GetNameInternal()); + + if (!current) { Entries.emplace_back(p->GetNameInternal(), p); - return; - } - } - + return; + } + } + for (unsigned i = 1;; ++i) { TString prefix = p->GetNameInternal(); - if (!prefix) { - prefix = "unnamed"; - } + if (!prefix) { + prefix = "unnamed"; + } TString name = ConcatStrings(prefix, "-", i); - - TValuePtr current = FindByName(name); - - if (!current) { + + TValuePtr current = FindByName(name); + + if (!current) { Entries.emplace_back(name, p); - return; - } - } - } - - size_t size() const { - return Entries.size(); - } - - bool operator!() const { - return size() == 0; - } - }; - - template <typename TSessionPtr> - struct TSessionValues: public TNamedValues<TSessionPtr> { - typedef TNamedValues<TSessionPtr> TBase; - + return; + } + } + } + + size_t size() const { + return Entries.size(); + } + + bool operator!() const { + return size() == 0; + } + }; + + template <typename TSessionPtr> + struct TSessionValues: public TNamedValues<TSessionPtr> { + typedef TNamedValues<TSessionPtr> TBase; + TVector<TString> GetNamesForQueue(TBusMessageQueue* queue) { TVector<TString> r; - for (unsigned i = 0; i < TBase::size(); ++i) { - if (TBase::Entries[i].second->GetQueue() == queue) { - r.push_back(TBase::Entries[i].first); - } - } - return r; - } - }; -} - -namespace { + for (unsigned i = 0; i < TBase::size(); ++i) { + if (TBase::Entries[i].second->GetQueue() == queue) { + r.push_back(TBase::Entries[i].first); + } + } + return r; + } + }; +} + +namespace { TString RootHref() { - return ConcatStrings("?"); - } - + return ConcatStrings("?"); + } + TString QueueHref(TStringBuf name) { - return ConcatStrings("?q=", name); - } - + return ConcatStrings("?q=", name); + } + TString ServerSessionHref(TStringBuf name) { - return ConcatStrings("?ss=", name); - } - + return ConcatStrings("?ss=", name); + } + TString ClientSessionHref(TStringBuf name) { - return ConcatStrings("?cs=", name); - } - + return ConcatStrings("?cs=", name); + } + TString OldModuleHref(TStringBuf name) { - return ConcatStrings("?om=", name); - } - - /* - static void RootLink() { - A(RootHref(), "root"); - } - */ - + return ConcatStrings("?om=", name); + } + + /* + static void RootLink() { + A(RootHref(), "root"); + } + */ + void QueueLink(TStringBuf name) { - A(QueueHref(name), name); - } - + A(QueueHref(name), name); + } + void ServerSessionLink(TStringBuf name) { - A(ServerSessionHref(name), name); - } - + A(ServerSessionHref(name), name); + } + void ClientSessionLink(TStringBuf name) { - A(ClientSessionHref(name), name); - } - + A(ClientSessionHref(name), name); + } + void OldModuleLink(TStringBuf name) { - A(OldModuleHref(name), name); - } - -} - -const unsigned char WWW_STATIC_DATA[] = { -#include "www_static.inc" -}; - + A(OldModuleHref(name), name); + } + +} + +const unsigned char WWW_STATIC_DATA[] = { +#include "www_static.inc" +}; + class TWwwStaticLoader: public TArchiveReader { -public: - TWwwStaticLoader() - : TArchiveReader(TBlob::NoCopy(WWW_STATIC_DATA, sizeof(WWW_STATIC_DATA))) - { - } -}; - -struct TBusWww::TImpl { - // TODO: use weak pointers - TNamedValues<TBusMessageQueuePtr> Queues; +public: + TWwwStaticLoader() + : TArchiveReader(TBlob::NoCopy(WWW_STATIC_DATA, sizeof(WWW_STATIC_DATA))) + { + } +}; + +struct TBusWww::TImpl { + // TODO: use weak pointers + TNamedValues<TBusMessageQueuePtr> Queues; TSessionValues<TIntrusivePtr<TBusClientSession>> ClientSessions; TSessionValues<TIntrusivePtr<TBusServerSession>> ServerSessions; - TSessionValues<TBusModuleInternalPtr> Modules; - - TMutex Mutex; - - void RegisterClientSession(TBusClientSessionPtr s) { + TSessionValues<TBusModuleInternalPtr> Modules; + + TMutex Mutex; + + void RegisterClientSession(TBusClientSessionPtr s) { Y_VERIFY(!!s); - TGuard<TMutex> g(Mutex); - ClientSessions.Add(s.Get()); - Queues.Add(s->GetQueue()); - } - - void RegisterServerSession(TBusServerSessionPtr s) { + TGuard<TMutex> g(Mutex); + ClientSessions.Add(s.Get()); + Queues.Add(s->GetQueue()); + } + + void RegisterServerSession(TBusServerSessionPtr s) { Y_VERIFY(!!s); - TGuard<TMutex> g(Mutex); - ServerSessions.Add(s.Get()); - Queues.Add(s->GetQueue()); - } - + TGuard<TMutex> g(Mutex); + ServerSessions.Add(s.Get()); + Queues.Add(s->GetQueue()); + } + void RegisterQueue(TBusMessageQueuePtr q) { Y_VERIFY(!!q); TGuard<TMutex> g(Mutex); Queues.Add(q); } - void RegisterModule(TBusModule* module) { + void RegisterModule(TBusModule* module) { Y_VERIFY(!!module); - TGuard<TMutex> g(Mutex); - - { + TGuard<TMutex> g(Mutex); + + { TVector<TBusClientSessionPtr> clientSessions = module->GetInternal()->GetClientSessionsInternal(); - for (unsigned i = 0; i < clientSessions.size(); ++i) { - RegisterClientSession(clientSessions[i]); - } - } - - { + for (unsigned i = 0; i < clientSessions.size(); ++i) { + RegisterClientSession(clientSessions[i]); + } + } + + { TVector<TBusServerSessionPtr> serverSessions = module->GetInternal()->GetServerSessionsInternal(); - for (unsigned i = 0; i < serverSessions.size(); ++i) { - RegisterServerSession(serverSessions[i]); - } - } - - Queues.Add(module->GetInternal()->GetQueue()); - Modules.Add(module->GetInternal()); - } - + for (unsigned i = 0; i < serverSessions.size(); ++i) { + RegisterServerSession(serverSessions[i]); + } + } + + Queues.Add(module->GetInternal()->GetQueue()); + Modules.Add(module->GetInternal()); + } + TString FindQueueNameBySessionName(TStringBuf sessionName, bool client) { - TIntrusivePtr<TBusClientSession> clientSession; - TIntrusivePtr<TBusServerSession> serverSession; - TBusSession* session; - if (client) { - clientSession = ClientSessions.FindByName(sessionName); - session = clientSession.Get(); - } else { - serverSession = ServerSessions.FindByName(sessionName); - session = serverSession.Get(); - } + TIntrusivePtr<TBusClientSession> clientSession; + TIntrusivePtr<TBusServerSession> serverSession; + TBusSession* session; + if (client) { + clientSession = ClientSessions.FindByName(sessionName); + session = clientSession.Get(); + } else { + serverSession = ServerSessions.FindByName(sessionName); + session = serverSession.Get(); + } Y_VERIFY(!!session); - return Queues.FindNameByPtr(session->GetQueue()); - } - - struct TRequest { - TImpl* const Outer; + return Queues.FindNameByPtr(session->GetQueue()); + } + + struct TRequest { + TImpl* const Outer; IOutputStream& Os; - const TCgiParameters& CgiParams; - const TOptionalParams& Params; - + const TCgiParameters& CgiParams; + const TOptionalParams& Params; + TRequest(TImpl* outer, IOutputStream& os, const TCgiParameters& cgiParams, const TOptionalParams& params) : Outer(outer) , Os(os) @@ -256,675 +256,675 @@ struct TBusWww::TImpl { , Params(params) { } - - void CrumbsParentLinks() { - for (unsigned i = 0; i < Params.ParentLinks.size(); ++i) { - const TLink& link = Params.ParentLinks[i]; - TTagGuard li("li"); - A(link.Href, link.Title); - } - } - - void Crumb(TStringBuf name, TStringBuf href = "") { - if (!!href) { - TTagGuard li("li"); - A(href, name); - } else { - LiWithClass("active", name); - } - } - - void BreadcrumbRoot() { - TTagGuard ol("ol", "breadcrumb"); - CrumbsParentLinks(); - Crumb("MessageBus"); - } - - void BreadcrumbQueue(TStringBuf queueName) { - TTagGuard ol("ol", "breadcrumb"); - CrumbsParentLinks(); - Crumb("MessageBus", RootHref()); - Crumb(ConcatStrings("queue ", queueName)); - } - - void BreadcrumbSession(TStringBuf sessionName, bool client) { + + void CrumbsParentLinks() { + for (unsigned i = 0; i < Params.ParentLinks.size(); ++i) { + const TLink& link = Params.ParentLinks[i]; + TTagGuard li("li"); + A(link.Href, link.Title); + } + } + + void Crumb(TStringBuf name, TStringBuf href = "") { + if (!!href) { + TTagGuard li("li"); + A(href, name); + } else { + LiWithClass("active", name); + } + } + + void BreadcrumbRoot() { + TTagGuard ol("ol", "breadcrumb"); + CrumbsParentLinks(); + Crumb("MessageBus"); + } + + void BreadcrumbQueue(TStringBuf queueName) { + TTagGuard ol("ol", "breadcrumb"); + CrumbsParentLinks(); + Crumb("MessageBus", RootHref()); + Crumb(ConcatStrings("queue ", queueName)); + } + + void BreadcrumbSession(TStringBuf sessionName, bool client) { TString queueName = Outer->FindQueueNameBySessionName(sessionName, client); - TStringBuf whatSession = client ? "client session" : "server session"; - - TTagGuard ol("ol", "breadcrumb"); - CrumbsParentLinks(); - Crumb("MessageBus", RootHref()); - Crumb(ConcatStrings("queue ", queueName), QueueHref(queueName)); - Crumb(ConcatStrings(whatSession, " ", sessionName)); - } - - void ServeSessionsOfQueue(TBusMessageQueuePtr queue, bool includeQueue) { + TStringBuf whatSession = client ? "client session" : "server session"; + + TTagGuard ol("ol", "breadcrumb"); + CrumbsParentLinks(); + Crumb("MessageBus", RootHref()); + Crumb(ConcatStrings("queue ", queueName), QueueHref(queueName)); + Crumb(ConcatStrings(whatSession, " ", sessionName)); + } + + void ServeSessionsOfQueue(TBusMessageQueuePtr queue, bool includeQueue) { TVector<TString> clientNames = Outer->ClientSessions.GetNamesForQueue(queue.Get()); TVector<TString> serverNames = Outer->ServerSessions.GetNamesForQueue(queue.Get()); TVector<TString> moduleNames = Outer->Modules.GetNamesForQueue(queue.Get()); - - TTagGuard table("table", "table table-condensed table-bordered"); - - { - TTagGuard colgroup("colgroup"); - TagWithClass("col", "col-md-2"); - TagWithClass("col", "col-md-2"); - TagWithClass("col", "col-md-8"); - } - - { - TTagGuard tr("tr"); - Th("What", "span2"); - Th("Name", "span2"); - Th("Status", "span6"); - } - - if (includeQueue) { + + TTagGuard table("table", "table table-condensed table-bordered"); + + { + TTagGuard colgroup("colgroup"); + TagWithClass("col", "col-md-2"); + TagWithClass("col", "col-md-2"); + TagWithClass("col", "col-md-8"); + } + + { + TTagGuard tr("tr"); + Th("What", "span2"); + Th("Name", "span2"); + Th("Status", "span6"); + } + + if (includeQueue) { TTagGuard tr1("tr"); - Td("queue"); - - { - TTagGuard td("td"); - QueueLink(Outer->Queues.FindNameByPtr(queue)); - } - - { + Td("queue"); + + { + TTagGuard td("td"); + QueueLink(Outer->Queues.FindNameByPtr(queue)); + } + + { TTagGuard tr2("td"); - Pre(queue->GetStatusSingleLine()); - } - } - - for (unsigned j = 0; j < clientNames.size(); ++j) { - TTagGuard tr("tr"); - Td("client session"); - - { - TTagGuard td("td"); - ClientSessionLink(clientNames[j]); - } - - { - TTagGuard td("td"); - Pre(Outer->ClientSessions.FindByName(clientNames[j])->GetStatusSingleLine()); - } - } - - for (unsigned j = 0; j < serverNames.size(); ++j) { - TTagGuard tr("tr"); - Td("server session"); - - { - TTagGuard td("td"); - ServerSessionLink(serverNames[j]); - } - - { - TTagGuard td("td"); - Pre(Outer->ServerSessions.FindByName(serverNames[j])->GetStatusSingleLine()); - } - } - - for (unsigned j = 0; j < moduleNames.size(); ++j) { - TTagGuard tr("tr"); - Td("module"); - - { - TTagGuard td("td"); - if (false) { - OldModuleLink(moduleNames[j]); - } else { - // TODO - Text(moduleNames[j]); - } - } - - { - TTagGuard td("td"); - Pre(Outer->Modules.FindByName(moduleNames[j])->GetStatusSingleLine()); - } - } - } - + Pre(queue->GetStatusSingleLine()); + } + } + + for (unsigned j = 0; j < clientNames.size(); ++j) { + TTagGuard tr("tr"); + Td("client session"); + + { + TTagGuard td("td"); + ClientSessionLink(clientNames[j]); + } + + { + TTagGuard td("td"); + Pre(Outer->ClientSessions.FindByName(clientNames[j])->GetStatusSingleLine()); + } + } + + for (unsigned j = 0; j < serverNames.size(); ++j) { + TTagGuard tr("tr"); + Td("server session"); + + { + TTagGuard td("td"); + ServerSessionLink(serverNames[j]); + } + + { + TTagGuard td("td"); + Pre(Outer->ServerSessions.FindByName(serverNames[j])->GetStatusSingleLine()); + } + } + + for (unsigned j = 0; j < moduleNames.size(); ++j) { + TTagGuard tr("tr"); + Td("module"); + + { + TTagGuard td("td"); + if (false) { + OldModuleLink(moduleNames[j]); + } else { + // TODO + Text(moduleNames[j]); + } + } + + { + TTagGuard td("td"); + Pre(Outer->Modules.FindByName(moduleNames[j])->GetStatusSingleLine()); + } + } + } + void ServeQueue(const TString& name) { - TBusMessageQueuePtr queue = Outer->Queues.FindByName(name); - - if (!queue) { - BootstrapError(ConcatStrings("queue not found by name: ", name)); - return; - } - - BreadcrumbQueue(name); - - TDivGuard container("container"); - - H1(ConcatStrings("MessageBus queue ", '"', name, '"')); - - TBusMessageQueueStatus status = queue->GetStatusRecordInternal(); - - Pre(status.PrintToString()); - - ServeSessionsOfQueue(queue, false); - - HnWithSmall(3, "Peak queue size", "(stored for an hour)"); - - { - TDivGuard div; - TDivGuard div2(TAttr("id", "queue-size-graph"), TAttr("style", "height: 300px")); - } - - { - TScriptFunctionGuard script; - - NJsonWriter::TBuf data(NJsonWriter::HEM_ESCAPE_HTML); - NJsonWriter::TBuf ticks(NJsonWriter::HEM_ESCAPE_HTML); - - const TExecutorHistory& history = status.ExecutorStatus.History; - - data.BeginList(); - ticks.BeginList(); - for (unsigned i = 0; i < history.HistoryRecords.size(); ++i) { - ui64 secondOfMinute = (history.FirstHistoryRecordSecond() + i) % 60; - ui64 minuteOfHour = (history.FirstHistoryRecordSecond() + i) / 60 % 60; - - unsigned printEach; - - if (history.HistoryRecords.size() <= 500) { - printEach = 1; - } else if (history.HistoryRecords.size() <= 1000) { - printEach = 2; - } else if (history.HistoryRecords.size() <= 3000) { - printEach = 6; - } else { - printEach = 12; - } - - if (secondOfMinute % printEach != 0) { - continue; - } - - ui32 max = 0; - for (unsigned j = 0; j < printEach; ++j) { - if (i < j) { - continue; - } - max = Max<ui32>(max, history.HistoryRecords[i - j].MaxQueueSize); - } - - data.BeginList(); - data.WriteString(ToString(i)); - data.WriteInt(max); - data.EndList(); - - // TODO: can be done with flot time plugin - if (history.HistoryRecords.size() <= 20) { - ticks.BeginList(); - ticks.WriteInt(i); - ticks.WriteString(ToString(secondOfMinute)); - ticks.EndList(); - } else if (history.HistoryRecords.size() <= 60) { - if (secondOfMinute % 5 == 0) { - ticks.BeginList(); - ticks.WriteInt(i); - ticks.WriteString(ToString(secondOfMinute)); - ticks.EndList(); - } - } else { - bool needTick; - if (history.HistoryRecords.size() <= 3 * 60) { - needTick = secondOfMinute % 15 == 0; - } else if (history.HistoryRecords.size() <= 7 * 60) { - needTick = secondOfMinute % 30 == 0; - } else if (history.HistoryRecords.size() <= 20 * 60) { - needTick = secondOfMinute == 0; - } else { - needTick = secondOfMinute == 0 && minuteOfHour % 5 == 0; - } - if (needTick) { - ticks.BeginList(); - ticks.WriteInt(i); + TBusMessageQueuePtr queue = Outer->Queues.FindByName(name); + + if (!queue) { + BootstrapError(ConcatStrings("queue not found by name: ", name)); + return; + } + + BreadcrumbQueue(name); + + TDivGuard container("container"); + + H1(ConcatStrings("MessageBus queue ", '"', name, '"')); + + TBusMessageQueueStatus status = queue->GetStatusRecordInternal(); + + Pre(status.PrintToString()); + + ServeSessionsOfQueue(queue, false); + + HnWithSmall(3, "Peak queue size", "(stored for an hour)"); + + { + TDivGuard div; + TDivGuard div2(TAttr("id", "queue-size-graph"), TAttr("style", "height: 300px")); + } + + { + TScriptFunctionGuard script; + + NJsonWriter::TBuf data(NJsonWriter::HEM_ESCAPE_HTML); + NJsonWriter::TBuf ticks(NJsonWriter::HEM_ESCAPE_HTML); + + const TExecutorHistory& history = status.ExecutorStatus.History; + + data.BeginList(); + ticks.BeginList(); + for (unsigned i = 0; i < history.HistoryRecords.size(); ++i) { + ui64 secondOfMinute = (history.FirstHistoryRecordSecond() + i) % 60; + ui64 minuteOfHour = (history.FirstHistoryRecordSecond() + i) / 60 % 60; + + unsigned printEach; + + if (history.HistoryRecords.size() <= 500) { + printEach = 1; + } else if (history.HistoryRecords.size() <= 1000) { + printEach = 2; + } else if (history.HistoryRecords.size() <= 3000) { + printEach = 6; + } else { + printEach = 12; + } + + if (secondOfMinute % printEach != 0) { + continue; + } + + ui32 max = 0; + for (unsigned j = 0; j < printEach; ++j) { + if (i < j) { + continue; + } + max = Max<ui32>(max, history.HistoryRecords[i - j].MaxQueueSize); + } + + data.BeginList(); + data.WriteString(ToString(i)); + data.WriteInt(max); + data.EndList(); + + // TODO: can be done with flot time plugin + if (history.HistoryRecords.size() <= 20) { + ticks.BeginList(); + ticks.WriteInt(i); + ticks.WriteString(ToString(secondOfMinute)); + ticks.EndList(); + } else if (history.HistoryRecords.size() <= 60) { + if (secondOfMinute % 5 == 0) { + ticks.BeginList(); + ticks.WriteInt(i); + ticks.WriteString(ToString(secondOfMinute)); + ticks.EndList(); + } + } else { + bool needTick; + if (history.HistoryRecords.size() <= 3 * 60) { + needTick = secondOfMinute % 15 == 0; + } else if (history.HistoryRecords.size() <= 7 * 60) { + needTick = secondOfMinute % 30 == 0; + } else if (history.HistoryRecords.size() <= 20 * 60) { + needTick = secondOfMinute == 0; + } else { + needTick = secondOfMinute == 0 && minuteOfHour % 5 == 0; + } + if (needTick) { + ticks.BeginList(); + ticks.WriteInt(i); ticks.WriteString(Sprintf(":%02u:%02u", (unsigned)minuteOfHour, (unsigned)secondOfMinute)); - ticks.EndList(); - } - } - } - ticks.EndList(); - data.EndList(); - - HtmlOutputStream() << " var data = " << data.Str() << ";\n"; - HtmlOutputStream() << " var ticks = " << ticks.Str() << ";\n"; - HtmlOutputStream() << " plotQueueSize('#queue-size-graph', data, ticks);\n"; - } - } - - void ServeSession(TStringBuf name, bool client) { - TIntrusivePtr<TBusClientSession> clientSession; - TIntrusivePtr<TBusServerSession> serverSession; - TBusSession* session; - TStringBuf whatSession; - if (client) { - whatSession = "client session"; - clientSession = Outer->ClientSessions.FindByName(name); - session = clientSession.Get(); - } else { - whatSession = "server session"; - serverSession = Outer->ServerSessions.FindByName(name); - session = serverSession.Get(); - } - if (!session) { - BootstrapError(ConcatStrings(whatSession, " not found by name: ", name)); - return; - } - - TSessionDumpStatus dumpStatus = session->GetStatusRecordInternal(); - - TBusMessageQueuePtr queue = session->GetQueue(); + ticks.EndList(); + } + } + } + ticks.EndList(); + data.EndList(); + + HtmlOutputStream() << " var data = " << data.Str() << ";\n"; + HtmlOutputStream() << " var ticks = " << ticks.Str() << ";\n"; + HtmlOutputStream() << " plotQueueSize('#queue-size-graph', data, ticks);\n"; + } + } + + void ServeSession(TStringBuf name, bool client) { + TIntrusivePtr<TBusClientSession> clientSession; + TIntrusivePtr<TBusServerSession> serverSession; + TBusSession* session; + TStringBuf whatSession; + if (client) { + whatSession = "client session"; + clientSession = Outer->ClientSessions.FindByName(name); + session = clientSession.Get(); + } else { + whatSession = "server session"; + serverSession = Outer->ServerSessions.FindByName(name); + session = serverSession.Get(); + } + if (!session) { + BootstrapError(ConcatStrings(whatSession, " not found by name: ", name)); + return; + } + + TSessionDumpStatus dumpStatus = session->GetStatusRecordInternal(); + + TBusMessageQueuePtr queue = session->GetQueue(); TString queueName = Outer->Queues.FindNameByPtr(session->GetQueue()); - - BreadcrumbSession(name, client); - - TDivGuard container("container"); - - H1(ConcatStrings("MessageBus ", whatSession, " ", '"', name, '"')); - - TBusMessageQueueStatus queueStatus = queue->GetStatusRecordInternal(); - - { - H3(ConcatStrings("queue ", queueName)); - Pre(queueStatus.PrintToString()); - } - - TSessionDumpStatus status = session->GetStatusRecordInternal(); - - if (status.Shutdown) { - BootstrapError("Session shut down"); - return; - } - - H3("Basic"); - Pre(status.Head); - - if (status.ConnectionStatusSummary.Server) { - H3("Acceptors"); - Pre(status.Acceptors); - } - - H3("Connections"); - Pre(status.ConnectionsSummary); - - { - TDivGuard div; - TTagGuard button("button", + + BreadcrumbSession(name, client); + + TDivGuard container("container"); + + H1(ConcatStrings("MessageBus ", whatSession, " ", '"', name, '"')); + + TBusMessageQueueStatus queueStatus = queue->GetStatusRecordInternal(); + + { + H3(ConcatStrings("queue ", queueName)); + Pre(queueStatus.PrintToString()); + } + + TSessionDumpStatus status = session->GetStatusRecordInternal(); + + if (status.Shutdown) { + BootstrapError("Session shut down"); + return; + } + + H3("Basic"); + Pre(status.Head); + + if (status.ConnectionStatusSummary.Server) { + H3("Acceptors"); + Pre(status.Acceptors); + } + + H3("Connections"); + Pre(status.ConnectionsSummary); + + { + TDivGuard div; + TTagGuard button("button", TAttr("type", "button"), TAttr("class", "btn"), TAttr("data-toggle", "collapse"), TAttr("data-target", "#connections")); - Text("Show connection details"); - } - { - TDivGuard div(TAttr("id", "connections"), TAttr("class", "collapse")); - Pre(status.Connections); - } - - H3("TBusSessionConfig"); - Pre(status.Config.PrintToString()); - - if (!client) { - H3("Message process time histogram"); - - const TDurationHistogram& h = + Text("Show connection details"); + } + { + TDivGuard div(TAttr("id", "connections"), TAttr("class", "collapse")); + Pre(status.Connections); + } + + H3("TBusSessionConfig"); + Pre(status.Config.PrintToString()); + + if (!client) { + H3("Message process time histogram"); + + const TDurationHistogram& h = dumpStatus.ConnectionStatusSummary.WriterStatus.Incremental.ProcessDurationHistogram; - - { - TDivGuard div; - TDivGuard div2(TAttr("id", "h"), TAttr("style", "height: 300px")); - } - - { - TScriptFunctionGuard script; - - NJsonWriter::TBuf buf(NJsonWriter::HEM_ESCAPE_HTML); - buf.BeginList(); - for (unsigned i = 0; i < h.Times.size(); ++i) { + + { + TDivGuard div; + TDivGuard div2(TAttr("id", "h"), TAttr("style", "height: 300px")); + } + + { + TScriptFunctionGuard script; + + NJsonWriter::TBuf buf(NJsonWriter::HEM_ESCAPE_HTML); + buf.BeginList(); + for (unsigned i = 0; i < h.Times.size(); ++i) { TString label = TDurationHistogram::LabelBefore(i); - buf.BeginList(); - buf.WriteString(label); - buf.WriteLongLong(h.Times[i]); - buf.EndList(); - } - buf.EndList(); - - HtmlOutputStream() << " var hist = " << buf.Str() << ";\n"; - HtmlOutputStream() << " plotHist('#h', hist);\n"; - } - } - } - - void ServeDefault() { - if (!Outer->Queues) { - BootstrapError("no queues"); - return; - } - - BreadcrumbRoot(); - - TDivGuard container("container"); - - H1("MessageBus queues"); - - for (unsigned i = 0; i < Outer->Queues.size(); ++i) { + buf.BeginList(); + buf.WriteString(label); + buf.WriteLongLong(h.Times[i]); + buf.EndList(); + } + buf.EndList(); + + HtmlOutputStream() << " var hist = " << buf.Str() << ";\n"; + HtmlOutputStream() << " plotHist('#h', hist);\n"; + } + } + } + + void ServeDefault() { + if (!Outer->Queues) { + BootstrapError("no queues"); + return; + } + + BreadcrumbRoot(); + + TDivGuard container("container"); + + H1("MessageBus queues"); + + for (unsigned i = 0; i < Outer->Queues.size(); ++i) { TString queueName = Outer->Queues.Entries[i].first; - TBusMessageQueuePtr queue = Outer->Queues.Entries[i].second; - - HnWithSmall(3, queueName, "(queue)"); - - ServeSessionsOfQueue(queue, true); - } - } - + TBusMessageQueuePtr queue = Outer->Queues.Entries[i].second; + + HnWithSmall(3, queueName, "(queue)"); + + ServeSessionsOfQueue(queue, true); + } + } + void WriteQueueSensors(NMonitoring::TDeprecatedJsonWriter& sj, TStringBuf queueName, TBusMessageQueue* queue) { - auto status = queue->GetStatusRecordInternal(); + auto status = queue->GetStatusRecordInternal(); sj.OpenMetric(); - sj.WriteLabels("mb_queue", queueName, "sensor", "WorkQueueSize"); - sj.WriteValue(status.ExecutorStatus.WorkQueueSize); + sj.WriteLabels("mb_queue", queueName, "sensor", "WorkQueueSize"); + sj.WriteValue(status.ExecutorStatus.WorkQueueSize); sj.CloseMetric(); - } - + } + void WriteMessageCounterSensors(NMonitoring::TDeprecatedJsonWriter& sj, TStringBuf labelName, TStringBuf sessionName, bool read, const TMessageCounter& counter) { - TStringBuf readOrWrite = read ? "read" : "write"; - + TStringBuf readOrWrite = read ? "read" : "write"; + sj.OpenMetric(); - sj.WriteLabels(labelName, sessionName, "mb_dir", readOrWrite, "sensor", "MessageBytes"); - sj.WriteValue(counter.BytesData); - sj.WriteModeDeriv(); + sj.WriteLabels(labelName, sessionName, "mb_dir", readOrWrite, "sensor", "MessageBytes"); + sj.WriteValue(counter.BytesData); + sj.WriteModeDeriv(); sj.CloseMetric(); - + sj.OpenMetric(); - sj.WriteLabels(labelName, sessionName, "mb_dir", readOrWrite, "sensor", "MessageCount"); - sj.WriteValue(counter.Count); - sj.WriteModeDeriv(); + sj.WriteLabels(labelName, sessionName, "mb_dir", readOrWrite, "sensor", "MessageCount"); + sj.WriteValue(counter.Count); + sj.WriteModeDeriv(); sj.CloseMetric(); - } - + } + void WriteSessionStatus(NMonitoring::TDeprecatedJsonWriter& sj, TStringBuf sessionName, bool client, TBusSession* session) { - TStringBuf labelName = client ? "mb_client_session" : "mb_server_session"; - - auto status = session->GetStatusRecordInternal(); - + TStringBuf labelName = client ? "mb_client_session" : "mb_server_session"; + + auto status = session->GetStatusRecordInternal(); + sj.OpenMetric(); - sj.WriteLabels(labelName, sessionName, "sensor", "InFlightCount"); - sj.WriteValue(status.Status.InFlightCount); + sj.WriteLabels(labelName, sessionName, "sensor", "InFlightCount"); + sj.WriteValue(status.Status.InFlightCount); sj.CloseMetric(); - + sj.OpenMetric(); - sj.WriteLabels(labelName, sessionName, "sensor", "InFlightSize"); - sj.WriteValue(status.Status.InFlightSize); + sj.WriteLabels(labelName, sessionName, "sensor", "InFlightSize"); + sj.WriteValue(status.Status.InFlightSize); sj.CloseMetric(); - + sj.OpenMetric(); - sj.WriteLabels(labelName, sessionName, "sensor", "SendQueueSize"); - sj.WriteValue(status.ConnectionStatusSummary.WriterStatus.SendQueueSize); + sj.WriteLabels(labelName, sessionName, "sensor", "SendQueueSize"); + sj.WriteValue(status.ConnectionStatusSummary.WriterStatus.SendQueueSize); sj.CloseMetric(); - - if (client) { + + if (client) { sj.OpenMetric(); - sj.WriteLabels(labelName, sessionName, "sensor", "AckMessagesSize"); - sj.WriteValue(status.ConnectionStatusSummary.WriterStatus.AckMessagesSize); + sj.WriteLabels(labelName, sessionName, "sensor", "AckMessagesSize"); + sj.WriteValue(status.ConnectionStatusSummary.WriterStatus.AckMessagesSize); sj.CloseMetric(); - } - - WriteMessageCounterSensors(sj, labelName, sessionName, false, + } + + WriteMessageCounterSensors(sj, labelName, sessionName, false, status.ConnectionStatusSummary.WriterStatus.Incremental.MessageCounter); - WriteMessageCounterSensors(sj, labelName, sessionName, true, + WriteMessageCounterSensors(sj, labelName, sessionName, true, status.ConnectionStatusSummary.ReaderStatus.Incremental.MessageCounter); - } - + } + void ServeSolomonJson(const TString& q, const TString& cs, const TString& ss) { Y_UNUSED(q); Y_UNUSED(cs); Y_UNUSED(ss); - bool all = q == "" && cs == "" && ss == ""; - + bool all = q == "" && cs == "" && ss == ""; + NMonitoring::TDeprecatedJsonWriter sj(&Os); - - sj.OpenDocument(); + + sj.OpenDocument(); sj.OpenMetrics(); - - for (unsigned i = 0; i < Outer->Queues.size(); ++i) { + + for (unsigned i = 0; i < Outer->Queues.size(); ++i) { TString queueName = Outer->Queues.Entries[i].first; - TBusMessageQueuePtr queue = Outer->Queues.Entries[i].second; - if (all || q == queueName) { - WriteQueueSensors(sj, queueName, &*queue); - } - + TBusMessageQueuePtr queue = Outer->Queues.Entries[i].second; + if (all || q == queueName) { + WriteQueueSensors(sj, queueName, &*queue); + } + TVector<TString> clientNames = Outer->ClientSessions.GetNamesForQueue(queue.Get()); TVector<TString> serverNames = Outer->ServerSessions.GetNamesForQueue(queue.Get()); TVector<TString> moduleNames = Outer->Modules.GetNamesForQueue(queue.Get()); - for (auto& sessionName : clientNames) { - if (all || cs == sessionName) { - auto session = Outer->ClientSessions.FindByName(sessionName); - WriteSessionStatus(sj, sessionName, true, &*session); - } - } - - for (auto& sessionName : serverNames) { - if (all || ss == sessionName) { - auto session = Outer->ServerSessions.FindByName(sessionName); - WriteSessionStatus(sj, sessionName, false, &*session); - } - } - } - + for (auto& sessionName : clientNames) { + if (all || cs == sessionName) { + auto session = Outer->ClientSessions.FindByName(sessionName); + WriteSessionStatus(sj, sessionName, true, &*session); + } + } + + for (auto& sessionName : serverNames) { + if (all || ss == sessionName) { + auto session = Outer->ServerSessions.FindByName(sessionName); + WriteSessionStatus(sj, sessionName, false, &*session); + } + } + } + sj.CloseMetrics(); - sj.CloseDocument(); - } - + sj.CloseDocument(); + } + void ServeStatic(IOutputStream& os, TStringBuf path) { if (path.EndsWith(".js")) { - os << HTTP_OK_JS; + os << HTTP_OK_JS; } else if (path.EndsWith(".png")) { - os << HTTP_OK_PNG; - } else { - os << HTTP_OK_BIN; - } + os << HTTP_OK_PNG; + } else { + os << HTTP_OK_BIN; + } TBlob blob = Singleton<TWwwStaticLoader>()->ObjectBlobByKey(TString("/") + TString(path)); - os.Write(blob.Data(), blob.Size()); - } - - void HeaderJsCss() { - LinkStylesheet("//yandex.st/bootstrap/3.0.2/css/bootstrap.css"); - LinkFavicon("?file=bus-ico.png"); - ScriptHref("//yandex.st/jquery/2.0.3/jquery.js"); - ScriptHref("//yandex.st/bootstrap/3.0.2/js/bootstrap.js"); - ScriptHref("//cdnjs.cloudflare.com/ajax/libs/flot/0.8.1/jquery.flot.min.js"); - ScriptHref("//cdnjs.cloudflare.com/ajax/libs/flot/0.8.1/jquery.flot.categories.min.js"); - ScriptHref("?file=messagebus.js"); - } - - void Serve() { - THtmlOutputStreamPushPop pp(&Os); - - TCgiParameters::const_iterator file = CgiParams.Find("file"); - if (file != CgiParams.end()) { - ServeStatic(Os, file->second); - return; - } - - bool solomonJson = false; - TCgiParameters::const_iterator fmt = CgiParams.Find("fmt"); - if (fmt != CgiParams.end()) { - if (fmt->second == "solomon-json") { - solomonJson = true; - } - } - - TCgiParameters::const_iterator cs = CgiParams.Find("cs"); - TCgiParameters::const_iterator ss = CgiParams.Find("ss"); - TCgiParameters::const_iterator q = CgiParams.Find("q"); - - if (solomonJson) { - Os << HTTP_OK_JSON; - + os.Write(blob.Data(), blob.Size()); + } + + void HeaderJsCss() { + LinkStylesheet("//yandex.st/bootstrap/3.0.2/css/bootstrap.css"); + LinkFavicon("?file=bus-ico.png"); + ScriptHref("//yandex.st/jquery/2.0.3/jquery.js"); + ScriptHref("//yandex.st/bootstrap/3.0.2/js/bootstrap.js"); + ScriptHref("//cdnjs.cloudflare.com/ajax/libs/flot/0.8.1/jquery.flot.min.js"); + ScriptHref("//cdnjs.cloudflare.com/ajax/libs/flot/0.8.1/jquery.flot.categories.min.js"); + ScriptHref("?file=messagebus.js"); + } + + void Serve() { + THtmlOutputStreamPushPop pp(&Os); + + TCgiParameters::const_iterator file = CgiParams.Find("file"); + if (file != CgiParams.end()) { + ServeStatic(Os, file->second); + return; + } + + bool solomonJson = false; + TCgiParameters::const_iterator fmt = CgiParams.Find("fmt"); + if (fmt != CgiParams.end()) { + if (fmt->second == "solomon-json") { + solomonJson = true; + } + } + + TCgiParameters::const_iterator cs = CgiParams.Find("cs"); + TCgiParameters::const_iterator ss = CgiParams.Find("ss"); + TCgiParameters::const_iterator q = CgiParams.Find("q"); + + if (solomonJson) { + Os << HTTP_OK_JSON; + TString qp = q != CgiParams.end() ? q->first : ""; TString csp = cs != CgiParams.end() ? cs->first : ""; TString ssp = ss != CgiParams.end() ? ss->first : ""; - ServeSolomonJson(qp, csp, ssp); - } else { - Os << HTTP_OK_HTML; - - Doctype(); - - TTagGuard html("html"); - { - TTagGuard head("head"); - - HeaderJsCss(); - // ✉ 🚌 - Title(TChars("MessageBus", false)); - } - - TTagGuard body("body"); - - if (cs != CgiParams.end()) { - ServeSession(cs->second, true); - } else if (ss != CgiParams.end()) { - ServeSession(ss->second, false); - } else if (q != CgiParams.end()) { - ServeQueue(q->second); - } else { - ServeDefault(); - } - } - } - }; - + ServeSolomonJson(qp, csp, ssp); + } else { + Os << HTTP_OK_HTML; + + Doctype(); + + TTagGuard html("html"); + { + TTagGuard head("head"); + + HeaderJsCss(); + // ✉ 🚌 + Title(TChars("MessageBus", false)); + } + + TTagGuard body("body"); + + if (cs != CgiParams.end()) { + ServeSession(cs->second, true); + } else if (ss != CgiParams.end()) { + ServeSession(ss->second, false); + } else if (q != CgiParams.end()) { + ServeQueue(q->second); + } else { + ServeDefault(); + } + } + } + }; + void ServeHttp(IOutputStream& os, const TCgiParameters& queryArgs, const TBusWww::TOptionalParams& params) { - TGuard<TMutex> g(Mutex); - - TRequest request(this, os, queryArgs, params); - - request.Serve(); - } -}; - -NBus::TBusWww::TBusWww() - : Impl(new TImpl) -{ -} - + TGuard<TMutex> g(Mutex); + + TRequest request(this, os, queryArgs, params); + + request.Serve(); + } +}; + +NBus::TBusWww::TBusWww() + : Impl(new TImpl) +{ +} + NBus::TBusWww::~TBusWww() { -} - +} + void NBus::TBusWww::RegisterClientSession(TBusClientSessionPtr s) { - Impl->RegisterClientSession(s); -} - + Impl->RegisterClientSession(s); +} + void TBusWww::RegisterServerSession(TBusServerSessionPtr s) { - Impl->RegisterServerSession(s); -} - + Impl->RegisterServerSession(s); +} + void TBusWww::RegisterQueue(TBusMessageQueuePtr q) { Impl->RegisterQueue(q); } -void TBusWww::RegisterModule(TBusModule* module) { - Impl->RegisterModule(module); -} - +void TBusWww::RegisterModule(TBusModule* module) { + Impl->RegisterModule(module); +} + void TBusWww::ServeHttp(IOutputStream& httpOutputStream, const TCgiParameters& queryArgs, const TBusWww::TOptionalParams& params) { - Impl->ServeHttp(httpOutputStream, queryArgs, params); -} - -struct TBusWwwHttpServer::TImpl: public THttpServer::ICallBack { - TIntrusivePtr<TBusWww> Www; - THttpServer HttpServer; - - static THttpServer::TOptions MakeHttpServerOptions(unsigned port) { + Impl->ServeHttp(httpOutputStream, queryArgs, params); +} + +struct TBusWwwHttpServer::TImpl: public THttpServer::ICallBack { + TIntrusivePtr<TBusWww> Www; + THttpServer HttpServer; + + static THttpServer::TOptions MakeHttpServerOptions(unsigned port) { Y_VERIFY(port > 0); - THttpServer::TOptions r; - r.Port = port; - return r; - } - - TImpl(TIntrusivePtr<TBusWww> www, unsigned port) - : Www(www) - , HttpServer(this, MakeHttpServerOptions(port)) - { - HttpServer.Start(); - } - - struct TClientRequestImpl: public TClientRequest { - TBusWwwHttpServer::TImpl* const Outer; - - TClientRequestImpl(TBusWwwHttpServer::TImpl* outer) - : Outer(outer) + THttpServer::TOptions r; + r.Port = port; + return r; + } + + TImpl(TIntrusivePtr<TBusWww> www, unsigned port) + : Www(www) + , HttpServer(this, MakeHttpServerOptions(port)) + { + HttpServer.Start(); + } + + struct TClientRequestImpl: public TClientRequest { + TBusWwwHttpServer::TImpl* const Outer; + + TClientRequestImpl(TBusWwwHttpServer::TImpl* outer) + : Outer(outer) { } - + bool Reply(void*) override { - Outer->ServeRequest(Input(), Output()); - return true; - } - }; - + Outer->ServeRequest(Input(), Output()); + return true; + } + }; + TString MakeSimpleResponse(unsigned code, TString text, TString content = "") { - if (!content) { - TStringStream contentSs; - contentSs << code << " " << text; - content = contentSs.Str(); - } - TStringStream ss; - ss << "HTTP/1.1 " + if (!content) { + TStringStream contentSs; + contentSs << code << " " << text; + content = contentSs.Str(); + } + TStringStream ss; + ss << "HTTP/1.1 " << code << " " << text << "\r\nConnection: Close\r\n\r\n" << content; - return ss.Str(); - } - - void ServeRequest(THttpInput& input, THttpOutput& output) { - TCgiParameters cgiParams; - try { - THttpRequestHeader header; - THttpHeaderParser parser; - parser.Init(&header); - if (parser.Execute(input.FirstLine()) < 0) { - HtmlOutputStream() << MakeSimpleResponse(400, "Bad request"); - return; - } - THttpURL url; - if (url.Parse(header.GetUrl()) != THttpURL::ParsedOK) { - HtmlOutputStream() << MakeSimpleResponse(400, "Invalid url"); - return; - } - cgiParams.Scan(url.Get(THttpURL::FieldQuery)); - - TBusWww::TOptionalParams params; + return ss.Str(); + } + + void ServeRequest(THttpInput& input, THttpOutput& output) { + TCgiParameters cgiParams; + try { + THttpRequestHeader header; + THttpHeaderParser parser; + parser.Init(&header); + if (parser.Execute(input.FirstLine()) < 0) { + HtmlOutputStream() << MakeSimpleResponse(400, "Bad request"); + return; + } + THttpURL url; + if (url.Parse(header.GetUrl()) != THttpURL::ParsedOK) { + HtmlOutputStream() << MakeSimpleResponse(400, "Invalid url"); + return; + } + cgiParams.Scan(url.Get(THttpURL::FieldQuery)); + + TBusWww::TOptionalParams params; //params.ParentLinks.emplace_back(); - //params.ParentLinks.back().Title = "temp"; - //params.ParentLinks.back().Href = "http://wiki.yandex-team.ru/"; - - Www->ServeHttp(output, cgiParams, params); - } catch (...) { - output << MakeSimpleResponse(500, "Exception", + //params.ParentLinks.back().Title = "temp"; + //params.ParentLinks.back().Href = "http://wiki.yandex-team.ru/"; + + Www->ServeHttp(output, cgiParams, params); + } catch (...) { + output << MakeSimpleResponse(500, "Exception", TString() + "Exception: " + CurrentExceptionMessage()); - } - } - + } + } + TClientRequest* CreateClient() override { - return new TClientRequestImpl(this); - } - + return new TClientRequestImpl(this); + } + ~TImpl() override { - HttpServer.Stop(); - } -}; - -NBus::TBusWwwHttpServer::TBusWwwHttpServer(TIntrusivePtr<TBusWww> www, unsigned port) - : Impl(new TImpl(www, port)) -{ -} - + HttpServer.Stop(); + } +}; + +NBus::TBusWwwHttpServer::TBusWwwHttpServer(TIntrusivePtr<TBusWww> www, unsigned port) + : Impl(new TImpl(www, port)) +{ +} + NBus::TBusWwwHttpServer::~TBusWwwHttpServer() { -} +} diff --git a/library/cpp/messagebus/www/www.h b/library/cpp/messagebus/www/www.h index 6e6f5f582c..6cd652b477 100644 --- a/library/cpp/messagebus/www/www.h +++ b/library/cpp/messagebus/www/www.h @@ -1,45 +1,45 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/ybus.h> #include <library/cpp/messagebus/oldmodule/module.h> #include <util/generic/ptr.h> #include <util/generic/string.h> #include <library/cpp/cgiparam/cgiparam.h> - -namespace NBus { + +namespace NBus { class TBusWww: public TAtomicRefCount<TBusWww> { public: struct TLink { TString Title; TString Href; }; - + struct TOptionalParams { TVector<TLink> ParentLinks; }; - + TBusWww(); ~TBusWww(); - + void RegisterClientSession(TBusClientSessionPtr); void RegisterServerSession(TBusServerSessionPtr); void RegisterQueue(TBusMessageQueuePtr); void RegisterModule(TBusModule*); - + void ServeHttp(IOutputStream& httpOutputStream, const TCgiParameters& queryArgs, const TOptionalParams& params = TOptionalParams()); - + struct TImpl; THolder<TImpl> Impl; }; - + class TBusWwwHttpServer { public: TBusWwwHttpServer(TIntrusivePtr<TBusWww> www, unsigned port); ~TBusWwwHttpServer(); - + struct TImpl; THolder<TImpl> Impl; }; - + } diff --git a/library/cpp/messagebus/www/ya.make b/library/cpp/messagebus/www/ya.make index f3e1300290..972390cea3 100644 --- a/library/cpp/messagebus/www/ya.make +++ b/library/cpp/messagebus/www/ya.make @@ -1,19 +1,19 @@ -LIBRARY() - +LIBRARY() + OWNER(g:messagebus) - -SRCS( - html_output.cpp - www.cpp -) - -ARCHIVE( - NAME www_static.inc - messagebus.js - bus-ico.png -) - -PEERDIR( + +SRCS( + html_output.cpp + www.cpp +) + +ARCHIVE( + NAME www_static.inc + messagebus.js + bus-ico.png +) + +PEERDIR( library/cpp/archive library/cpp/cgiparam library/cpp/html/pcdata @@ -24,6 +24,6 @@ PEERDIR( library/cpp/messagebus/oldmodule library/cpp/monlib/deprecated/json library/cpp/uri -) - -END() +) + +END() diff --git a/library/cpp/messagebus/ya.make b/library/cpp/messagebus/ya.make index a8fdf6de92..e13cf06dea 100644 --- a/library/cpp/messagebus/ya.make +++ b/library/cpp/messagebus/ya.make @@ -1,7 +1,7 @@ LIBRARY() OWNER(g:messagebus) - + IF (SANITIZER_TYPE == "undefined") NO_SANITIZE() ENDIF() diff --git a/library/cpp/messagebus/ybus.h b/library/cpp/messagebus/ybus.h index dcd06824f0..de21ad8521 100644 --- a/library/cpp/messagebus/ybus.h +++ b/library/cpp/messagebus/ybus.h @@ -1,20 +1,20 @@ #pragma once /// Asynchronous Messaging Library implements framework for sending and -/// receiving messages between loosely connected processes. +/// receiving messages between loosely connected processes. #include "coreconn.h" -#include "defs.h" -#include "handler.h" -#include "handler_impl.h" -#include "local_flags.h" +#include "defs.h" +#include "handler.h" +#include "handler_impl.h" +#include "local_flags.h" #include "locator.h" -#include "message.h" +#include "message.h" #include "message_status.h" #include "network.h" #include "queue_config.h" #include "remote_connection_status.h" -#include "session.h" +#include "session.h" #include "session_config.h" #include "socket_addr.h" @@ -24,8 +24,8 @@ #include <library/cpp/codecs/codecs.h> #include <util/generic/array_ref.h> -#include <util/generic/buffer.h> -#include <util/generic/noncopyable.h> +#include <util/generic/buffer.h> +#include <util/generic/noncopyable.h> #include <util/generic/ptr.h> #include <util/stream/input.h> #include <util/system/atomic.h> @@ -49,7 +49,7 @@ namespace NBus { /// and destination server. Protocol object is reponsible for serializing in-memory /// message and reply into the wire, retuning name of the service and resource /// distribution key for given protocol. - + /// Protocol object should transparently handle messages and replies. /// This is interface only class, actuall instances of the protocols /// should be created using templates inhereted from this base class. @@ -105,24 +105,24 @@ namespace NBus { public: TBusSyncSourceSession(TIntrusivePtr< ::NBus::NPrivate::TBusSyncSourceSessionImpl> session); ~TBusSyncSourceSession(); - + void Shutdown(); - + TBusMessage* SendSyncMessage(TBusMessage* pMessage, EMessageStatus& status, const TNetAddr* addr = nullptr); - + int RegisterService(const char* hostname, TBusKey start = YBUS_KEYMIN, TBusKey end = YBUS_KEYMAX, EIpVersion ipVersion = EIP_VERSION_4); - + int GetInFlight(); - + const TBusProtocol* GetProto() const; const TBusClientSession* GetBusClientSessionWorkaroundDoNotUse() const; // It's for TLoadBalancedProtocol::GetDestination() function that really needs TBusClientSession* unlike all other protocols. Look at review 32425 (http://rb.yandex-team.ru/arc/r/32425/) for more information. private: TIntrusivePtr< ::NBus::NPrivate::TBusSyncSourceSessionImpl> Session; }; - + using TBusSyncClientSessionPtr = TIntrusivePtr<TBusSyncSourceSession>; - + /////////////////////////////////////////////////////////////////// /// \brief Main message queue object, need one per application class TBusMessageQueue: public TAtomicRefCount<TBusMessageQueue> { @@ -132,7 +132,7 @@ namespace NBus { friend struct ::NBus::NPrivate::TBusSessionImpl; friend class ::NBus::NPrivate::TAcceptor; friend struct ::NBus::TBusServerSession; - + private: const TBusQueueConfig Config; TMutex Lock; @@ -144,16 +144,16 @@ namespace NBus { TAtomic Running; TSystemEvent ShutdownComplete; - + private: /// constructor is protected, used NBus::CreateMessageQueue() to create a instance TBusMessageQueue(const TBusQueueConfig& config, NActor::TExecutorPtr executor, TBusLocator* locator, const char* name); - + public: TString GetNameInternal() const; ~TBusMessageQueue(); - + void Stop(); bool IsRunning(); @@ -161,17 +161,17 @@ namespace NBus { void EnqueueWork(TArrayRef< ::NActor::IWorkItem* const> w) { WorkQueue->EnqueueWork(w); } - + ::NActor::TExecutor* GetExecutor() { return WorkQueue.Get(); } - + TString GetStatus(ui16 flags = YBUS_STATUS_CONNS) const; // without sessions NPrivate::TBusMessageQueueStatus GetStatusRecordInternal() const; TString GetStatusSelf() const; TString GetStatusSingleLine() const; - + TBusLocator* GetLocator() const { return Locator.Get(); } diff --git a/library/cpp/monlib/deprecated/json/ut/ya.make b/library/cpp/monlib/deprecated/json/ut/ya.make index 556b0c8291..18315993b5 100644 --- a/library/cpp/monlib/deprecated/json/ut/ya.make +++ b/library/cpp/monlib/deprecated/json/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/monlib/deprecated/json) - + OWNER( jamel g:solomon ) - -SRCS( + +SRCS( writer_ut.cpp -) - -END() +) + +END() diff --git a/library/cpp/monlib/deprecated/json/writer.cpp b/library/cpp/monlib/deprecated/json/writer.cpp index 9917288687..a581f2e07a 100644 --- a/library/cpp/monlib/deprecated/json/writer.cpp +++ b/library/cpp/monlib/deprecated/json/writer.cpp @@ -1,36 +1,36 @@ #include "writer.h" - + namespace NMonitoring { TDeprecatedJsonWriter::TDeprecatedJsonWriter(IOutputStream* out) : JsonWriter(out, false) , State(STATE_ROOT) { } - + void TDeprecatedJsonWriter::TransitionState(EState current, EState next) { if (State != current) { ythrow yexception() << "wrong state"; } State = next; } - + void TDeprecatedJsonWriter::OpenDocument() { TransitionState(STATE_ROOT, STATE_DOCUMENT); JsonWriter.OpenMap(); - } - + } + void TDeprecatedJsonWriter::CloseDocument() { TransitionState(STATE_DOCUMENT, STATE_ROOT); JsonWriter.CloseMap(); JsonWriter.Flush(); } - + void TDeprecatedJsonWriter::OpenCommonLabels() { TransitionState(STATE_DOCUMENT, STATE_COMMON_LABELS); JsonWriter.Write("commonLabels"); JsonWriter.OpenMap(); } - + void TDeprecatedJsonWriter::CloseCommonLabels() { TransitionState(STATE_COMMON_LABELS, STATE_DOCUMENT); JsonWriter.CloseMap(); @@ -51,50 +51,50 @@ namespace NMonitoring { TransitionState(STATE_METRICS, STATE_DOCUMENT); JsonWriter.CloseArray(); } - + void TDeprecatedJsonWriter::OpenMetric() { TransitionState(STATE_METRICS, STATE_METRIC); JsonWriter.OpenMap(); } - + void TDeprecatedJsonWriter::CloseMetric() { TransitionState(STATE_METRIC, STATE_METRICS); JsonWriter.CloseMap(); } - + void TDeprecatedJsonWriter::OpenLabels() { TransitionState(STATE_METRIC, STATE_LABELS); JsonWriter.Write("labels"); JsonWriter.OpenMap(); } - + void TDeprecatedJsonWriter::CloseLabels() { TransitionState(STATE_LABELS, STATE_METRIC); JsonWriter.CloseMap(); } - + void TDeprecatedJsonWriter::WriteLabel(TStringBuf name, TStringBuf value) { TransitionState(STATE_LABELS, STATE_LABELS); JsonWriter.Write(name, value); } - + void TDeprecatedJsonWriter::WriteModeDeriv() { TransitionState(STATE_METRIC, STATE_METRIC); JsonWriter.Write("mode", "deriv"); } - + void TDeprecatedJsonWriter::WriteValue(long long value) { TransitionState(STATE_METRIC, STATE_METRIC); JsonWriter.Write("value", value); } - + void TDeprecatedJsonWriter::WriteDoubleValue(double value) { TransitionState(STATE_METRIC, STATE_METRIC); JsonWriter.Write("value", value); } - + void TDeprecatedJsonWriter::WriteTs(ui64 ts) { TransitionState(STATE_METRIC, STATE_METRIC); JsonWriter.Write("ts", ts); } -} +} diff --git a/library/cpp/monlib/deprecated/json/writer.h b/library/cpp/monlib/deprecated/json/writer.h index 2ea8ce40c1..183288143c 100644 --- a/library/cpp/monlib/deprecated/json/writer.h +++ b/library/cpp/monlib/deprecated/json/writer.h @@ -1,7 +1,7 @@ -#pragma once - +#pragma once + #include <library/cpp/json/json_writer.h> - + namespace NMonitoring { /** * Deprecated writer of Solomon JSON format @@ -12,24 +12,24 @@ namespace NMonitoring { * particular format. */ class TDeprecatedJsonWriter { - private: - NJson::TJsonWriter JsonWriter; - enum EState { - STATE_ROOT, - STATE_DOCUMENT, + private: + NJson::TJsonWriter JsonWriter; + enum EState { + STATE_ROOT, + STATE_DOCUMENT, STATE_COMMON_LABELS, STATE_METRICS, STATE_METRIC, - STATE_LABELS, - }; - EState State; + STATE_LABELS, + }; + EState State; - public: + public: explicit TDeprecatedJsonWriter(IOutputStream* out); - - void OpenDocument(); - void CloseDocument(); - + + void OpenDocument(); + void CloseDocument(); + void OpenCommonLabels(); void CloseCommonLabels(); @@ -37,40 +37,40 @@ namespace NMonitoring { void OpenMetrics(); void CloseMetrics(); - + void OpenMetric(); void CloseMetric(); - - void OpenLabels(); - void CloseLabels(); - - void WriteLabel(TStringBuf name, TStringBuf value); - - template <typename... T> - void WriteLabels(T... pairs) { - OpenLabels(); - WriteLabelsInner(pairs...); - CloseLabels(); - } - - void WriteModeDeriv(); - - void WriteValue(long long value); - void WriteDoubleValue(double d); + + void OpenLabels(); + void CloseLabels(); + + void WriteLabel(TStringBuf name, TStringBuf value); + + template <typename... T> + void WriteLabels(T... pairs) { + OpenLabels(); + WriteLabelsInner(pairs...); + CloseLabels(); + } + + void WriteModeDeriv(); + + void WriteValue(long long value); + void WriteDoubleValue(double d); void WriteTs(ui64 ts); - private: - void WriteLabelsInner(TStringBuf name, TStringBuf value) { - WriteLabel(name, value); - } - + private: + void WriteLabelsInner(TStringBuf name, TStringBuf value) { + WriteLabel(name, value); + } + template <typename... T> - void WriteLabelsInner(TStringBuf name, TStringBuf value, T... pairs) { - WriteLabel(name, value); - WriteLabelsInner(pairs...); - } - - inline void TransitionState(EState current, EState next); - }; + void WriteLabelsInner(TStringBuf name, TStringBuf value, T... pairs) { + WriteLabel(name, value); + WriteLabelsInner(pairs...); + } + + inline void TransitionState(EState current, EState next); + }; } diff --git a/library/cpp/monlib/deprecated/json/writer_ut.cpp b/library/cpp/monlib/deprecated/json/writer_ut.cpp index cd5c4e4d78..1f9fc8f393 100644 --- a/library/cpp/monlib/deprecated/json/writer_ut.cpp +++ b/library/cpp/monlib/deprecated/json/writer_ut.cpp @@ -1,32 +1,32 @@ #include "writer.h" #include <library/cpp/testing/unittest/registar.h> - + using namespace NMonitoring; - + Y_UNIT_TEST_SUITE(JsonWriterTests) { Y_UNIT_TEST(One) { - TStringStream ss; + TStringStream ss; TDeprecatedJsonWriter w(&ss); - w.OpenDocument(); + w.OpenDocument(); w.OpenMetrics(); - - for (int i = 0; i < 5; ++i) { + + for (int i = 0; i < 5; ++i) { w.OpenMetric(); - w.OpenLabels(); + w.OpenLabels(); w.WriteLabel("user", TString("") + (char)('a' + i)); w.WriteLabel("name", "NWrites"); - w.CloseLabels(); - if (i % 2 == 0) { - w.WriteModeDeriv(); - } - w.WriteValue(10l); + w.CloseLabels(); + if (i % 2 == 0) { + w.WriteModeDeriv(); + } + w.WriteValue(10l); w.CloseMetric(); - } - + } + w.CloseMetrics(); - w.CloseDocument(); - - //Cout << ss.Str() << "\n"; - } -} + w.CloseDocument(); + + //Cout << ss.Str() << "\n"; + } +} diff --git a/library/cpp/monlib/deprecated/json/ya.make b/library/cpp/monlib/deprecated/json/ya.make index cf49d7367b..0ca903ee62 100644 --- a/library/cpp/monlib/deprecated/json/ya.make +++ b/library/cpp/monlib/deprecated/json/ya.make @@ -1,26 +1,26 @@ -LIBRARY() - +LIBRARY() + # Deprecated writer of Solomon JSON format -# https://wiki.yandex-team.ru/solomon/api/dataformat/json +# https://wiki.yandex-team.ru/solomon/api/dataformat/json # # This writer will be deleted soon, so please consider to use # high level library library/cpp/monlib/encode which is decoupled from the # particular format. - + OWNER( jamel g:solomon ) - + SRCS( writer.h writer.cpp ) -PEERDIR( +PEERDIR( library/cpp/json -) - -END() +) + +END() RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/monlib/dynamic_counters/counters.cpp b/library/cpp/monlib/dynamic_counters/counters.cpp index 8a73c388eb..3635d87d0d 100644 --- a/library/cpp/monlib/dynamic_counters/counters.cpp +++ b/library/cpp/monlib/dynamic_counters/counters.cpp @@ -2,10 +2,10 @@ #include <library/cpp/monlib/service/pages/templates.h> -#include <util/generic/cast.h> - -using namespace NMonitoring; - +#include <util/generic/cast.h> + +using namespace NMonitoring; + namespace { TDynamicCounters* AsDynamicCounters(const TIntrusivePtr<TCountableBase>& ptr) { return dynamic_cast<TDynamicCounters*>(ptr.Get()); @@ -53,7 +53,7 @@ TDynamicCounters::TDynamicCounters(EVisibility vis) TDynamicCounters::~TDynamicCounters() { } - + TDynamicCounters::TCounterPtr TDynamicCounters::GetExpiringCounter(const TString& value, bool derivative, EVisibility vis) { return GetExpiringNamedCounter("sensor", value, derivative, vis); } @@ -68,8 +68,8 @@ TDynamicCounters::TCounterPtr TDynamicCounters::GetCounter(const TString& value, TDynamicCounters::TCounterPtr TDynamicCounters::GetNamedCounter(const TString& name, const TString& value, bool derivative, EVisibility vis) { return AsCounterRef(GetNamedCounterImpl<false, TCounterForPtr>(name, value, derivative, vis)); -} - +} + THistogramPtr TDynamicCounters::GetHistogram(const TString& value, IHistogramCollectorPtr collector, bool derivative, EVisibility vis) { return GetNamedHistogram("sensor", value, std::move(collector), derivative, vis); } @@ -126,7 +126,7 @@ TIntrusivePtr<TDynamicCounters> TDynamicCounters::GetSubgroup(const TString& nam } } return res; -} +} TIntrusivePtr<TDynamicCounters> TDynamicCounters::FindSubgroup(const TString& name, const TString& value) const { TReadGuard g(Lock); @@ -186,7 +186,7 @@ void TDynamicCounters::RegisterSubgroup(const TString& name, const TString& valu void TDynamicCounters::OutputHtml(IOutputStream& os) const { HTML(os) { PRE() { - OutputPlainText(os); + OutputPlainText(os); } } } @@ -199,7 +199,7 @@ void TDynamicCounters::EnumerateSubgroups(const std::function<void(const TString } } } - + void TDynamicCounters::OutputPlainText(IOutputStream& os, const TString& indent) const { auto snap = ReadSnapshot(); // mark private records in plain text output @@ -232,18 +232,18 @@ void TDynamicCounters::OutputPlainText(IOutputStream& os, const TString& indent) } os << ": " << snapshot->Value(i) << '\n'; } - } - } - + } + } + for (const auto& [key, value] : snap) { if (const auto subgroup = AsDynamicCounters(value)) { - os << "\n"; + os << "\n"; os << indent << key.LabelName << "=" << key.LabelValue << ":\n"; subgroup->OutputPlainText(os, indent + INDENT); - } - } -} - + } + } +} + void TDynamicCounters::Accept(const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const { if (!IsVisible(Visibility(), consumer.Visibility())) { return; diff --git a/library/cpp/monlib/dynamic_counters/counters.h b/library/cpp/monlib/dynamic_counters/counters.h index 548e92ef39..dc178cfbe0 100644 --- a/library/cpp/monlib/dynamic_counters/counters.h +++ b/library/cpp/monlib/dynamic_counters/counters.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include <library/cpp/monlib/counters/counters.h> #include <library/cpp/monlib/metrics/histogram_collector.h> @@ -7,14 +7,14 @@ #include <library/cpp/containers/stack_vector/stack_vec.h> #include <util/generic/cast.h> -#include <util/generic/map.h> -#include <util/generic/ptr.h> +#include <util/generic/map.h> +#include <util/generic/ptr.h> #include <util/string/cast.h> #include <util/system/rwlock.h> #include <functional> - -namespace NMonitoring { + +namespace NMonitoring { struct TCounterForPtr; struct TDynamicCounters; struct ICountableConsumer; @@ -44,8 +44,8 @@ namespace NMonitoring { protected: EVisibility Visibility_{EVisibility::Unspecified}; - }; - + }; + inline bool IsVisible(TCountableBase::EVisibility myLevel, TCountableBase::EVisibility consumerLevel) { if (myLevel == TCountableBase::EVisibility::Private && consumerLevel != TCountableBase::EVisibility::Private) { @@ -114,8 +114,8 @@ namespace NMonitoring { using TDeprecatedCounter::operator-=; using TDeprecatedCounter::operator=; using TDeprecatedCounter::operator!; - }; - + }; + struct TExpiringCounter: public TCounterForPtr { explicit TExpiringCounter(bool derivative = false, EVisibility vis = EVisibility::Public) : TCounterForPtr{derivative} @@ -185,34 +185,34 @@ namespace NMonitoring { #pragma warning(pop) #endif - struct TDynamicCounters; - - typedef TIntrusivePtr<TDynamicCounters> TDynamicCounterPtr; - struct TDynamicCounters: public TCountableBase { - public: + struct TDynamicCounters; + + typedef TIntrusivePtr<TDynamicCounters> TDynamicCounterPtr; + struct TDynamicCounters: public TCountableBase { + public: using TCounterPtr = TIntrusivePtr<TCounterForPtr>; using TOnLookupPtr = void (*)(const char *methodName, const TString &name, const TString &value); - - private: + + private: TRWMutex Lock; TCounterPtr LookupCounter; // Counts lookups by name TOnLookupPtr OnLookup = nullptr; // Called on each lookup if not nullptr, intended for lightweight tracing. - - typedef TIntrusivePtr<TCountableBase> TCountablePtr; - - struct TChildId { + + typedef TIntrusivePtr<TCountableBase> TCountablePtr; + + struct TChildId { TString LabelName; TString LabelValue; TChildId() { } TChildId(const TString& labelName, const TString& labelValue) - : LabelName(labelName) - , LabelValue(labelValue) - { - } + : LabelName(labelName) + , LabelValue(labelValue) + { + } auto AsTuple() const { return std::make_tuple(std::cref(LabelName), std::cref(LabelValue)); - } + } friend bool operator <(const TChildId& x, const TChildId& y) { return x.AsTuple() < y.AsTuple(); } @@ -222,8 +222,8 @@ namespace NMonitoring { friend bool operator !=(const TChildId& x, const TChildId& y) { return x.AsTuple() != y.AsTuple(); } - }; - + }; + using TCounters = TMap<TChildId, TCountablePtr>; using TLabels = TVector<TChildId>; @@ -231,7 +231,7 @@ namespace NMonitoring { mutable TCounters Counters; mutable TAtomic ExpiringCount = 0; - public: + public: TDynamicCounters(TCountableBase::EVisibility visibility = TCountableBase::EVisibility::Public); TDynamicCounters(const TDynamicCounters *origin) @@ -240,7 +240,7 @@ namespace NMonitoring { {} ~TDynamicCounters() override; - + // This counter allows to track lookups by name within the whole subtree void SetLookupCounter(TCounterPtr lookupCounter) { TWriteGuard g(Lock); @@ -328,7 +328,7 @@ namespace NMonitoring { void RemoveCounter(const TString &value); void RemoveNamedCounter(const TString& name, const TString &value); - + TIntrusivePtr<TDynamicCounters> GetSubgroup(const TString& name, const TString& value); TIntrusivePtr<TDynamicCounters> FindSubgroup(const TString& name, const TString& value) const; void RemoveSubgroup(const TString& name, const TString& value); @@ -348,12 +348,12 @@ namespace NMonitoring { // mostly for debugging purposes -- use accept with encoder instead void OutputPlainText(IOutputStream& os, const TString& indent = "") const; - + void Accept( const TString& labelName, const TString& labelValue, ICountableConsumer& consumer) const override; - private: + private: TCounters Resign() { TCounters counters; TWriteGuard g(Lock); @@ -369,6 +369,6 @@ namespace NMonitoring { template <class TCounterType> TCountablePtr FindNamedCounterImpl(const TString& name, const TString& value) const; - }; - -} + }; + +} diff --git a/library/cpp/monlib/dynamic_counters/page.cpp b/library/cpp/monlib/dynamic_counters/page.cpp index 2ddc74bd88..5124a47bb3 100644 --- a/library/cpp/monlib/dynamic_counters/page.cpp +++ b/library/cpp/monlib/dynamic_counters/page.cpp @@ -1,14 +1,14 @@ #include "page.h" #include "encode.h" - + #include <library/cpp/monlib/service/pages/templates.h> #include <library/cpp/string_utils/quote/quote.h> #include <util/string/split.h> #include <util/system/tls.h> - -using namespace NMonitoring; - + +using namespace NMonitoring; + namespace { Y_POD_STATIC_THREAD(TDynamicCounters*) currentCounters(nullptr); @@ -74,10 +74,10 @@ void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { if (!format) { currentCounters = counters.Get(); - THtmlMonPage::Output(request); + THtmlMonPage::Output(request); currentCounters = nullptr; return; - } + } IOutputStream& out = request.Output(); if (*format == EFormat::JSON) { @@ -93,7 +93,7 @@ void TDynamicCountersPage::Output(NMonitoring::IMonHttpRequest& request) { auto encoder = CreateEncoder(&out, *format, visibility); counters->Accept(TString(), TString(), *encoder); out.Flush(); -} +} void TDynamicCountersPage::HandleAbsentSubgroup(IMonHttpRequest& request) { if (UnknownGroupPolicy == EUnknownGroupPolicy::Error) { @@ -110,7 +110,7 @@ void TDynamicCountersPage::BeforePre(IMonHttpRequest& request) { HTML(out) { DIV() { out << "<a href='" << request.GetPath() << "/json'>Counters as JSON</a>"; - out << " for <a href='https://wiki.yandex-team.ru/solomon/'>Solomon</a>"; + out << " for <a href='https://wiki.yandex-team.ru/solomon/'>Solomon</a>"; } H5() { @@ -131,10 +131,10 @@ void TDynamicCountersPage::BeforePre(IMonHttpRequest& request) { } } } - + void TDynamicCountersPage::OutputText(IOutputStream& out, IMonHttpRequest&) { currentCounters->OutputPlainText(out); -} +} void TDynamicCountersPage::SetUnknownGroupPolicy(EUnknownGroupPolicy value) { UnknownGroupPolicy = value; diff --git a/library/cpp/monlib/dynamic_counters/page.h b/library/cpp/monlib/dynamic_counters/page.h index c318ded739..1f0ef6a5ea 100644 --- a/library/cpp/monlib/dynamic_counters/page.h +++ b/library/cpp/monlib/dynamic_counters/page.h @@ -1,24 +1,24 @@ -#pragma once - +#pragma once + #include "counters.h" #include <library/cpp/monlib/service/pages/pre_mon_page.h> -#include <util/generic/ptr.h> - +#include <util/generic/ptr.h> + #include <functional> - -namespace NMonitoring { + +namespace NMonitoring { enum class EUnknownGroupPolicy { Error, // send 404 Ignore, // send 204 }; - struct TDynamicCountersPage: public TPreMonPage { + struct TDynamicCountersPage: public TPreMonPage { public: using TOutputCallback = std::function<void()>; - private: + private: const TIntrusivePtr<TDynamicCounters> Counters; TOutputCallback OutputCallback; EUnknownGroupPolicy UnknownGroupPolicy {EUnknownGroupPolicy::Error}; @@ -26,19 +26,19 @@ namespace NMonitoring { private: void HandleAbsentSubgroup(IMonHttpRequest& request); - public: + public: TDynamicCountersPage(const TString& path, const TString& title, TIntrusivePtr<TDynamicCounters> counters, TOutputCallback outputCallback = nullptr) : TPreMonPage(path, title) - , Counters(counters) + , Counters(counters) , OutputCallback(outputCallback) { } - + void Output(NMonitoring::IMonHttpRequest& request) override; - + void BeforePre(NMonitoring::IMonHttpRequest& request) override; void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; @@ -47,4 +47,4 @@ namespace NMonitoring { /// If set to Ignore, responds with 204 if the requested subgroup is not found void SetUnknownGroupPolicy(EUnknownGroupPolicy value); }; -} +} diff --git a/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto b/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto index 275903bd9e..fd23eb372b 100644 --- a/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto +++ b/library/cpp/monlib/encode/legacy_protobuf/protos/metric_meta.proto @@ -1,37 +1,37 @@ import "google/protobuf/descriptor.proto"; - + package NMonProto; - + option java_package = "ru.yandex.monlib.proto"; option java_outer_classname = "MetricMetaProto"; - + enum EMetricType { GAUGE = 1; RATE = 2; -} - +} + enum EMemOnly { - DEFAULT = 0; - STORE = 1; - MEM_ONLY = 2; -} - + DEFAULT = 0; + STORE = 1; + MEM_ONLY = 2; +} + message TMetricMeta { optional EMetricType Type = 1; - optional bool Path = 2; - optional string Keys = 3; - optional bool MemOnly = 4; - optional bool IgnorePath = 5; + optional bool Path = 2; + optional string Keys = 3; + optional bool MemOnly = 4; + optional bool IgnorePath = 5; optional string CustomPath = 6; -} - -enum THistogramBase { - MICROSECOND = 3; - MILLISECOND = 6; - SECOND = 9; - MINUTE = 12; - HOUR = 15; -} +} + +enum THistogramBase { + MICROSECOND = 3; + MILLISECOND = 6; + SECOND = 9; + MINUTE = 12; + HOUR = 15; +} message THistogramEntry { optional uint64 Multiplier = 1; @@ -40,34 +40,34 @@ message THistogramEntry { message THistogram { optional THistogramBase Base = 1; - optional string BaseStr = 2; + optional string BaseStr = 2; repeated THistogramEntry Entries = 5; } - -// field of this type is recognized by Solomon + +// field of this type is recognized by Solomon message TExtraLabelMetrics { - optional string labelName = 1; - - message TValue { - optional string labelValue = 1; - // used only if != 0 - optional uint64 labelValueUint = 21; - - optional uint64 longValue = 2; - optional double doubleValue = 3; - optional THistogram histogramValue = 4; - + optional string labelName = 1; + + message TValue { + optional string labelValue = 1; + // used only if != 0 + optional uint64 labelValueUint = 21; + + optional uint64 longValue = 2; + optional double doubleValue = 3; + optional THistogram histogramValue = 4; + optional EMetricType type = 7; optional EMemOnly memOnly = 8; - optional bool dropHost = 9; - + optional bool dropHost = 9; + repeated TExtraLabelMetrics children = 17; - } - - repeated TValue values = 2; -} - -extend google.protobuf.FieldOptions { + } + + repeated TValue values = 2; +} + +extend google.protobuf.FieldOptions { optional TMetricMeta Metric = 1719; -} - +} + diff --git a/library/cpp/monlib/messagebus/mon_messagebus.cpp b/library/cpp/monlib/messagebus/mon_messagebus.cpp index 5d6c7b0214..355b4386cd 100644 --- a/library/cpp/monlib/messagebus/mon_messagebus.cpp +++ b/library/cpp/monlib/messagebus/mon_messagebus.cpp @@ -1,11 +1,11 @@ #include <library/cpp/messagebus/www/www.h> - -#include "mon_messagebus.h" - -using namespace NMonitoring; - + +#include "mon_messagebus.h" + +using namespace NMonitoring; + void TBusNgMonPage::Output(NMonitoring::IMonHttpRequest& request) { - NBus::TBusWww::TOptionalParams params; + NBus::TBusWww::TOptionalParams params; params.ParentLinks.push_back(NBus::TBusWww::TLink{"/", request.GetServiceTitle()}); BusWww->ServeHttp(request.Output(), request.GetParams(), params); -} +} diff --git a/library/cpp/monlib/messagebus/mon_messagebus.h b/library/cpp/monlib/messagebus/mon_messagebus.h index dbbc618d7b..e1fa73c69f 100644 --- a/library/cpp/monlib/messagebus/mon_messagebus.h +++ b/library/cpp/monlib/messagebus/mon_messagebus.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include <library/cpp/messagebus/ybus.h> #include <library/cpp/messagebus/www/www.h> - + #include <library/cpp/monlib/service/pages/mon_page.h> - -namespace NMonitoring { + +namespace NMonitoring { template <class TBusSmth> class TBusSmthMonPage: public NMonitoring::IMonPage { private: TBusSmth* Smth; - + public: explicit TBusSmthMonPage(const TString& name, const TString& title, TBusSmth* smth) : IMonPage("msgbus/" + name, title) @@ -26,14 +26,14 @@ namespace NMonitoring { request.Output() << "</pre>"; } }; - + using TBusQueueMonPage = TBusSmthMonPage<NBus::TBusMessageQueue>; using TBusModuleMonPage = TBusSmthMonPage<NBus::TBusModule>; - + class TBusNgMonPage: public NMonitoring::IMonPage { public: TIntrusivePtr<NBus::TBusWww> BusWww; - + public: TBusNgMonPage() : IMonPage("messagebus", "MessageBus") diff --git a/library/cpp/monlib/messagebus/mon_service_messagebus.cpp b/library/cpp/monlib/messagebus/mon_service_messagebus.cpp index 34ea259033..4dd144ebe8 100644 --- a/library/cpp/monlib/messagebus/mon_service_messagebus.cpp +++ b/library/cpp/monlib/messagebus/mon_service_messagebus.cpp @@ -1,8 +1,8 @@ -#include "mon_service_messagebus.h" - -using namespace NMonitoring; - +#include "mon_service_messagebus.h" + +using namespace NMonitoring; + TMonServiceMessageBus::TMonServiceMessageBus(ui16 port, const TString& title) - : TMonService2(port, title) + : TMonService2(port, title) { } diff --git a/library/cpp/monlib/messagebus/mon_service_messagebus.h b/library/cpp/monlib/messagebus/mon_service_messagebus.h index d3e05886d4..fe791e8a9b 100644 --- a/library/cpp/monlib/messagebus/mon_service_messagebus.h +++ b/library/cpp/monlib/messagebus/mon_service_messagebus.h @@ -1,20 +1,20 @@ -#pragma once - +#pragma once + #include "mon_messagebus.h" #include <library/cpp/monlib/service/monservice.h> -#include <util/system/mutex.h> - -namespace NMonitoring { +#include <util/system/mutex.h> + +namespace NMonitoring { class TMonServiceMessageBus: public TMonService2 { private: TMutex Mtx; TIntrusivePtr<NMonitoring::TBusNgMonPage> BusNgMonPage; - + public: TMonServiceMessageBus(ui16 port, const TString& title); - + private: NBus::TBusWww* RegisterBusNgMonPage() { TGuard<TMutex> g(Mtx); @@ -24,20 +24,20 @@ namespace NMonitoring { } return BusNgMonPage->BusWww.Get(); } - + public: void RegisterClientSession(NBus::TBusClientSessionPtr clientSession) { RegisterBusNgMonPage()->RegisterClientSession(clientSession); - } - + } + void RegisterServerSession(NBus::TBusServerSessionPtr serverSession) { RegisterBusNgMonPage()->RegisterServerSession(serverSession); } - + void RegisterQueue(NBus::TBusMessageQueuePtr queue) { RegisterBusNgMonPage()->RegisterQueue(queue); } - + void RegisterModule(NBus::TBusModule* module) { RegisterBusNgMonPage()->RegisterModule(module); } diff --git a/library/cpp/monlib/messagebus/ya.make b/library/cpp/monlib/messagebus/ya.make index fb85222a61..a0b5362296 100644 --- a/library/cpp/monlib/messagebus/ya.make +++ b/library/cpp/monlib/messagebus/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - +LIBRARY() + OWNER(g:solomon) - -SRCS( - mon_messagebus.cpp - mon_service_messagebus.cpp -) - -PEERDIR( + +SRCS( + mon_messagebus.cpp + mon_service_messagebus.cpp +) + +PEERDIR( library/cpp/messagebus library/cpp/messagebus/www library/cpp/monlib/dynamic_counters -) - -END() +) + +END() diff --git a/library/cpp/monlib/service/mon_service_http_request.cpp b/library/cpp/monlib/service/mon_service_http_request.cpp index b8d1d27b98..5d805631d9 100644 --- a/library/cpp/monlib/service/mon_service_http_request.cpp +++ b/library/cpp/monlib/service/mon_service_http_request.cpp @@ -1,8 +1,8 @@ #include "mon_service_http_request.h" -#include "monservice.h" - -using namespace NMonitoring; - +#include "monservice.h" + +using namespace NMonitoring; + IMonHttpRequest::~IMonHttpRequest() { } @@ -10,9 +10,9 @@ TMonService2HttpRequest::~TMonService2HttpRequest() { } TString TMonService2HttpRequest::GetServiceTitle() const { - return MonService->GetTitle(); -} - + return MonService->GetTitle(); +} + IOutputStream& TMonService2HttpRequest::Output() { return *Out; } diff --git a/library/cpp/monlib/service/mon_service_http_request.h b/library/cpp/monlib/service/mon_service_http_request.h index 8dd75044cf..b4f2f8f0c5 100644 --- a/library/cpp/monlib/service/mon_service_http_request.h +++ b/library/cpp/monlib/service/mon_service_http_request.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include "service.h" -#include <util/stream/output.h> - -namespace NMonitoring { +#include <util/stream/output.h> + +namespace NMonitoring { class TMonService2; class IMonPage; - + // XXX: IHttpRequest is already taken struct IMonHttpRequest { virtual ~IMonHttpRequest(); @@ -40,12 +40,12 @@ namespace NMonitoring { IMonPage* const MonPage; const TString PathInfo; TMonService2HttpRequest* const Parent; - + TMonService2HttpRequest( IOutputStream* out, const IHttpRequest* httpRequest, - TMonService2* monService, IMonPage* monPage, + TMonService2* monService, IMonPage* monPage, const TString& pathInfo, - TMonService2HttpRequest* parent) + TMonService2HttpRequest* parent) : Out(out) , HttpRequest(httpRequest) , MonService(monService) @@ -54,9 +54,9 @@ namespace NMonitoring { , Parent(parent) { } - + ~TMonService2HttpRequest() override; - + IOutputStream& Output() override; HTTP_METHOD GetMethod() const override; TStringBuf GetPath() const override; @@ -67,7 +67,7 @@ namespace NMonitoring { TStringBuf GetPostContent() const override { return HttpRequest->GetPostContent(); } - + TStringBuf GetHeader(TStringBuf name) const override; TStringBuf GetCookie(TStringBuf name) const override; const THttpHeaders& GetHeaders() const override; @@ -86,5 +86,5 @@ namespace NMonitoring { TString GetServiceTitle() const override; }; - + } diff --git a/library/cpp/monlib/service/monservice.cpp b/library/cpp/monlib/service/monservice.cpp index 49a1c941e4..d1b9cda1d2 100644 --- a/library/cpp/monlib/service/monservice.cpp +++ b/library/cpp/monlib/service/monservice.cpp @@ -1,17 +1,17 @@ -#include "monservice.h" +#include "monservice.h" #include <library/cpp/malloc/api/malloc.h> #include <library/cpp/string_utils/base64/base64.h> #include <library/cpp/svnversion/svnversion.h> -#include <util/generic/map.h> +#include <util/generic/map.h> #include <util/generic/ptr.h> -#include <util/system/hostname.h> +#include <util/system/hostname.h> #include <google/protobuf/text_format.h> - -using namespace NMonitoring; - + +using namespace NMonitoring; + TMonService2::TMonService2(ui16 port, const TString& host, ui32 threads, const TString& title, THolder<IAuthProvider> auth) : TMonService2(HttpServerOptions(port, host, threads), title, std::move(auth)) { @@ -19,15 +19,15 @@ TMonService2::TMonService2(ui16 port, const TString& host, ui32 threads, const T TMonService2::TMonService2(const THttpServerOptions& options, const TString& title, THolder<IAuthProvider> auth) : NMonitoring::TMtHttpServer(options, std::bind(&TMonService2::ServeRequest, this, std::placeholders::_1, std::placeholders::_2)) - , Title(title) - , IndexMonPage(new TIndexMonPage("", Title)) + , Title(title) + , IndexMonPage(new TIndexMonPage("", Title)) , AuthProvider_{std::move(auth)} -{ +{ Y_VERIFY(!!title); time_t t = time(nullptr); - ctime_r(&t, StartTime); -} - + ctime_r(&t, StartTime); +} + TMonService2::TMonService2(const THttpServerOptions& options, TSimpleSharedPtr<IThreadPool> pool, const TString& title, THolder<IAuthProvider> auth) : NMonitoring::TMtHttpServer(options, std::bind(&TMonService2::ServeRequest, this, std::placeholders::_1, std::placeholders::_2), std::move(pool)) , Title(title) @@ -50,33 +50,33 @@ TMonService2::TMonService2(ui16 port, const TString& title, THolder<IAuthProvide } void TMonService2::OutputIndex(IOutputStream& out) { - IndexMonPage->OutputIndex(out, true); -} - + IndexMonPage->OutputIndex(out, true); +} + void TMonService2::OutputIndexPage(IOutputStream& out) { - out << HTTPOKHTML; - out << "<html>\n"; - IndexMonPage->OutputHead(out); - OutputIndexBody(out); - out << "</html>\n"; -} - + out << HTTPOKHTML; + out << "<html>\n"; + IndexMonPage->OutputHead(out); + OutputIndexBody(out); + out << "</html>\n"; +} + void TMonService2::OutputIndexBody(IOutputStream& out) { - out << "<body>\n"; - - // part of common navbar - out << "<ol class='breadcrumb'>\n"; - out << "<li class='active'>" << Title << "</li>\n"; - out << "</ol>\n"; - - out << "<div class='container'>\n" - << "<h2>" << Title << "</h2>\n"; - OutputIndex(out); - out - << "<div>\n" - << "</body>\n"; -} - + out << "<body>\n"; + + // part of common navbar + out << "<ol class='breadcrumb'>\n"; + out << "<li class='active'>" << Title << "</li>\n"; + out << "</ol>\n"; + + out << "<div class='container'>\n" + << "<h2>" << Title << "</h2>\n"; + OutputIndex(out); + out + << "<div>\n" + << "</body>\n"; +} + void TMonService2::ServeRequest(IOutputStream& out, const NMonitoring::IHttpRequest& request) { TString path = request.GetPath(); Y_VERIFY(path.StartsWith('/')); @@ -95,34 +95,34 @@ void TMonService2::ServeRequest(IOutputStream& out, const NMonitoring::IHttpRequ } } - if (path == "/") { - OutputIndexPage(out); - } else { - TMonService2HttpRequest monService2HttpRequest( + if (path == "/") { + OutputIndexPage(out); + } else { + TMonService2HttpRequest monService2HttpRequest( &out, &request, this, IndexMonPage.Get(), path, nullptr); - IndexMonPage->Output(monService2HttpRequest); - } -} - + IndexMonPage->Output(monService2HttpRequest); + } +} + void TMonService2::Register(IMonPage* page) { - IndexMonPage->Register(page); -} - + IndexMonPage->Register(page); +} + void TMonService2::Register(TMonPagePtr page) { IndexMonPage->Register(std::move(page)); } TIndexMonPage* TMonService2::RegisterIndexPage(const TString& path, const TString& title) { return IndexMonPage->RegisterIndexPage(path, title); -} - +} + IMonPage* TMonService2::FindPage(const TString& relativePath) { - return IndexMonPage->FindPage(relativePath); -} - + return IndexMonPage->FindPage(relativePath); +} + TIndexMonPage* TMonService2::FindIndexPage(const TString& relativePath) { - return IndexMonPage->FindIndexPage(relativePath); -} + return IndexMonPage->FindIndexPage(relativePath); +} void TMonService2::SortPages() { IndexMonPage->SortPages(); diff --git a/library/cpp/monlib/service/monservice.h b/library/cpp/monlib/service/monservice.h index 288c6089a1..8f5e52fcdb 100644 --- a/library/cpp/monlib/service/monservice.h +++ b/library/cpp/monlib/service/monservice.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + #include "service.h" #include "auth.h" #include "mon_service_http_request.h" @@ -7,18 +7,18 @@ #include <library/cpp/monlib/service/pages/index_mon_page.h> #include <library/cpp/monlib/service/pages/mon_page.h> -#include <util/system/progname.h> +#include <util/system/progname.h> #include <functional> - -namespace NMonitoring { + +namespace NMonitoring { class TMonService2: public TMtHttpServer { protected: const TString Title; char StartTime[26]; TIntrusivePtr<TIndexMonPage> IndexMonPage; THolder<IAuthProvider> AuthProvider_; - + public: static THttpServerOptions HttpServerOptions(ui16 port, const TString& host, ui32 threads) { THttpServerOptions opts(port); @@ -32,7 +32,7 @@ namespace NMonitoring { opts.EnableRejectExcessConnections(true); return opts; } - + static THttpServerOptions HttpServerOptions(ui16 port, ui32 threads) { return HttpServerOptions(port, TString(), threads); } @@ -43,31 +43,31 @@ namespace NMonitoring { explicit TMonService2(ui16 port, const TString& host, ui32 threads, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); explicit TMonService2(const THttpServerOptions& options, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); explicit TMonService2(const THttpServerOptions& options, TSimpleSharedPtr<IThreadPool> pool, const TString& title = GetProgramName(), THolder<IAuthProvider> auth = nullptr); - + ~TMonService2() override { } - + const char* GetStartTime() const { return StartTime; } - + const TString& GetTitle() const { return Title; } - + virtual void ServeRequest(IOutputStream& out, const NMonitoring::IHttpRequest& request); virtual void OutputIndex(IOutputStream& out); virtual void OutputIndexPage(IOutputStream& out); virtual void OutputIndexBody(IOutputStream& out); - + void Register(IMonPage* page); void Register(TMonPagePtr page); TIndexMonPage* RegisterIndexPage(const TString& path, const TString& title); - + IMonPage* FindPage(const TString& relativePath); TIndexMonPage* FindIndexPage(const TString& relativePath); void SortPages(); }; - + } diff --git a/library/cpp/monlib/service/pages/diag_mon_page.cpp b/library/cpp/monlib/service/pages/diag_mon_page.cpp index cf073e9224..2493ff4fba 100644 --- a/library/cpp/monlib/service/pages/diag_mon_page.cpp +++ b/library/cpp/monlib/service/pages/diag_mon_page.cpp @@ -1,9 +1,9 @@ -#include "diag_mon_page.h" - -using namespace NMonitoring; - +#include "diag_mon_page.h" + +using namespace NMonitoring; + void TDiagMonPage::OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest& request) { out << "uri: " << request.GetUri() << "\n"; out << "path: " << request.GetPath() << "\n"; out << "path info: " << request.GetPathInfo() << "\n"; -} +} diff --git a/library/cpp/monlib/service/pages/diag_mon_page.h b/library/cpp/monlib/service/pages/diag_mon_page.h index bb16711700..761194d4ec 100644 --- a/library/cpp/monlib/service/pages/diag_mon_page.h +++ b/library/cpp/monlib/service/pages/diag_mon_page.h @@ -1,16 +1,16 @@ -#pragma once - -#include "pre_mon_page.h" - -namespace NMonitoring { +#pragma once + +#include "pre_mon_page.h" + +namespace NMonitoring { // internal diagnostics page struct TDiagMonPage: public TPreMonPage { TDiagMonPage() : TPreMonPage("diag", "Diagnostics Page") { } - + void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; }; - + } diff --git a/library/cpp/monlib/service/pages/html_mon_page.cpp b/library/cpp/monlib/service/pages/html_mon_page.cpp index 7b2a7cb135..eb4eb3b66c 100644 --- a/library/cpp/monlib/service/pages/html_mon_page.cpp +++ b/library/cpp/monlib/service/pages/html_mon_page.cpp @@ -1,20 +1,20 @@ -#include "html_mon_page.h" - +#include "html_mon_page.h" + #include <library/cpp/monlib/service/pages/templates.h> - -using namespace NMonitoring; - + +using namespace NMonitoring; + void THtmlMonPage::Output(NMonitoring::IMonHttpRequest& request) { IOutputStream& out = request.Output(); - out << HTTPOKHTML; + out << HTTPOKHTML; HTML(out) { - out << "<!DOCTYPE html>\n"; + out << "<!DOCTYPE html>\n"; HTML_TAG() { HEAD() { - if (!!Title) { - out << "<title>" << Title << "</title>\n"; - } + if (!!Title) { + out << "<title>" << Title << "</title>\n"; + } out << "<link rel='stylesheet' href='https://yastatic.net/bootstrap/3.3.1/css/bootstrap.min.css'>\n"; out << "<script language='javascript' type='text/javascript' src='https://yastatic.net/jquery/2.1.3/jquery.min.js'></script>\n"; out << "<script language='javascript' type='text/javascript' src='https://yastatic.net/bootstrap/3.3.1/js/bootstrap.min.js'></script>\n"; @@ -35,17 +35,17 @@ void THtmlMonPage::Output(NMonitoring::IMonHttpRequest& request) { } BODY() { OutputNavBar(out); - + DIV_CLASS("container") { - if (!!Title) { - out << "<h2>" << Title << "</h2>"; - } - OutputContent(request); + if (!!Title) { + out << "<h2>" << Title << "</h2>"; + } + OutputContent(request); } } } } -} +} void THtmlMonPage::NotFound(NMonitoring::IMonHttpRequest& request) const { IOutputStream& out = request.Output(); diff --git a/library/cpp/monlib/service/pages/html_mon_page.h b/library/cpp/monlib/service/pages/html_mon_page.h index a3e260866e..e87c53b62b 100644 --- a/library/cpp/monlib/service/pages/html_mon_page.h +++ b/library/cpp/monlib/service/pages/html_mon_page.h @@ -1,8 +1,8 @@ -#pragma once - -#include "mon_page.h" - -namespace NMonitoring { +#pragma once + +#include "mon_page.h" + +namespace NMonitoring { struct THtmlMonPage: public IMonPage { THtmlMonPage(const TString& path, const TString& title = TString(), @@ -11,9 +11,9 @@ namespace NMonitoring { , OutputTableSorterJsCss(outputTableSorterJsCss) { } - + void Output(NMonitoring::IMonHttpRequest& request) override; - + void NotFound(NMonitoring::IMonHttpRequest& request) const; void NoContent(NMonitoring::IMonHttpRequest& request) const; @@ -21,5 +21,5 @@ namespace NMonitoring { bool OutputTableSorterJsCss; }; - + } diff --git a/library/cpp/monlib/service/pages/index_mon_page.cpp b/library/cpp/monlib/service/pages/index_mon_page.cpp index 108c1945cd..83ff8b529a 100644 --- a/library/cpp/monlib/service/pages/index_mon_page.cpp +++ b/library/cpp/monlib/service/pages/index_mon_page.cpp @@ -1,72 +1,72 @@ #include "index_mon_page.h" -#include <util/generic/cast.h> +#include <util/generic/cast.h> #include <util/string/ascii.h> - -using namespace NMonitoring; - + +using namespace NMonitoring; + void TIndexMonPage::OutputIndexPage(IMonHttpRequest& request) { request.Output() << HTTPOKHTML; request.Output() << "<html>\n"; OutputHead(request.Output()); - OutputBody(request); + OutputBody(request); request.Output() << "</html>\n"; -} - +} + void TIndexMonPage::Output(IMonHttpRequest& request) { TStringBuf pathInfo = request.GetPathInfo(); if (pathInfo.empty() || pathInfo == TStringBuf("/")) { - OutputIndexPage(request); - return; - } - + OutputIndexPage(request); + return; + } + Y_VERIFY(pathInfo.StartsWith('/')); - - TMonPagePtr found; - // analogous to CGI PATH_INFO - { - TGuard<TMutex> g(Mtx); + + TMonPagePtr found; + // analogous to CGI PATH_INFO + { + TGuard<TMutex> g(Mtx); TStringBuf pathTmp = request.GetPathInfo(); - for (;;) { - TPagesByPath::iterator i = PagesByPath.find(pathTmp); - if (i != PagesByPath.end()) { - found = i->second; + for (;;) { + TPagesByPath::iterator i = PagesByPath.find(pathTmp); + if (i != PagesByPath.end()) { + found = i->second; pathInfo = request.GetPathInfo().substr(pathTmp.size()); Y_VERIFY(pathInfo.empty() || pathInfo.StartsWith('/')); - break; - } - size_t slash = pathTmp.find_last_of('/'); + break; + } + size_t slash = pathTmp.find_last_of('/'); Y_VERIFY(slash != TString::npos); - pathTmp = pathTmp.substr(0, slash); - if (!pathTmp) { - break; - } - } - } + pathTmp = pathTmp.substr(0, slash); + if (!pathTmp) { + break; + } + } + } if (found) { THolder<IMonHttpRequest> child(request.MakeChild(found.Get(), TString{pathInfo})); found->Output(*child); - } else { + } else { request.Output() << HTTPNOTFOUND; - } -} - + } +} + void TIndexMonPage::OutputIndex(IOutputStream& out, bool pathEndsWithSlash) { - TGuard<TMutex> g(Mtx); + TGuard<TMutex> g(Mtx); for (auto& Page : Pages) { IMonPage* page = Page.Get(); - if (page->IsInIndex()) { + if (page->IsInIndex()) { TString pathToDir = ""; - if (!pathEndsWithSlash) { - pathToDir = this->GetPath() + "/"; - } - out << "<a href='" << pathToDir << page->GetPath() << "'>" << page->GetTitle() << "</a><br/>\n"; - } - } -} - + if (!pathEndsWithSlash) { + pathToDir = this->GetPath() + "/"; + } + out << "<a href='" << pathToDir << page->GetPath() << "'>" << page->GetTitle() << "</a><br/>\n"; + } + } +} + void TIndexMonPage::Register(TMonPagePtr page) { - TGuard<TMutex> g(Mtx); + TGuard<TMutex> g(Mtx); auto insres = PagesByPath.insert(std::make_pair("/" + page->GetPath(), page)); if (insres.second) { // new unique page just inserted, update Pages @@ -81,8 +81,8 @@ void TIndexMonPage::Register(TMonPagePtr page) { insres.first->second = page; } page->Parent = this; -} - +} + TIndexMonPage* TIndexMonPage::RegisterIndexPage(const TString& path, const TString& title) { TGuard<TMutex> g(Mtx); TIndexMonPage* page = VerifyDynamicCast<TIndexMonPage*>(FindPage(path)); @@ -92,50 +92,50 @@ TIndexMonPage* TIndexMonPage::RegisterIndexPage(const TString& path, const TStri page = new TIndexMonPage(path, title); Register(page); return VerifyDynamicCast<TIndexMonPage*>(page); -} - +} + IMonPage* TIndexMonPage::FindPage(const TString& relativePath) { - TGuard<TMutex> g(Mtx); - + TGuard<TMutex> g(Mtx); + Y_VERIFY(!relativePath.StartsWith('/')); - TPagesByPath::iterator i = PagesByPath.find("/" + relativePath); - if (i == PagesByPath.end()) { + TPagesByPath::iterator i = PagesByPath.find("/" + relativePath); + if (i == PagesByPath.end()) { return nullptr; - } else { - return i->second.Get(); - } -} - + } else { + return i->second.Get(); + } +} + TIndexMonPage* TIndexMonPage::FindIndexPage(const TString& relativePath) { - return VerifyDynamicCast<TIndexMonPage*>(FindPage(relativePath)); -} - + return VerifyDynamicCast<TIndexMonPage*>(FindPage(relativePath)); +} + void TIndexMonPage::OutputCommonJsCss(IOutputStream& out) { out << "<link rel='stylesheet' href='https://yastatic.net/bootstrap/3.3.1/css/bootstrap.min.css'>\n"; out << "<script language='javascript' type='text/javascript' src='https://yastatic.net/jquery/2.1.3/jquery.min.js'></script>\n"; out << "<script language='javascript' type='text/javascript' src='https://yastatic.net/bootstrap/3.3.1/js/bootstrap.min.js'></script>\n"; -} - +} + void TIndexMonPage::OutputHead(IOutputStream& out) { - out << "<head>\n"; - OutputCommonJsCss(out); - out << "<title>" << Title << "</title>\n"; - out << "</head>\n"; -} - + out << "<head>\n"; + OutputCommonJsCss(out); + out << "<title>" << Title << "</title>\n"; + out << "</head>\n"; +} + void TIndexMonPage::OutputBody(IMonHttpRequest& req) { auto& out = req.Output(); out << "<body>\n"; - - // part of common navbar + + // part of common navbar OutputNavBar(out); - + out << "<div class='container'>\n" << "<h2>" << Title << "</h2>\n"; OutputIndex(out, req.GetPathInfo().EndsWith('/')); out << "<div>\n" - << "</body>\n"; -} + << "</body>\n"; +} void TIndexMonPage::SortPages() { TGuard<TMutex> g(Mtx); diff --git a/library/cpp/monlib/service/pages/index_mon_page.h b/library/cpp/monlib/service/pages/index_mon_page.h index 9a1ff45145..bf514a3105 100644 --- a/library/cpp/monlib/service/pages/index_mon_page.h +++ b/library/cpp/monlib/service/pages/index_mon_page.h @@ -1,38 +1,38 @@ -#pragma once - -#include "mon_page.h" - -namespace NMonitoring { +#pragma once + +#include "mon_page.h" + +namespace NMonitoring { struct TIndexMonPage: public IMonPage { TMutex Mtx; typedef TVector<TMonPagePtr> TPages; TPages Pages; typedef THashMap<TString, TMonPagePtr> TPagesByPath; TPagesByPath PagesByPath; - + TIndexMonPage(const TString& path, const TString& title) : IMonPage(path, title) { } - + ~TIndexMonPage() override { } - + void Output(IMonHttpRequest& request) override; void OutputIndexPage(IMonHttpRequest& request); virtual void OutputIndex(IOutputStream& out, bool pathEndsWithSlash); virtual void OutputCommonJsCss(IOutputStream& out); void OutputHead(IOutputStream& out); void OutputBody(IMonHttpRequest& out); - + void Register(TMonPagePtr page); TIndexMonPage* RegisterIndexPage(const TString& path, const TString& title); IMonPage* FindPage(const TString& relativePath); TIndexMonPage* FindIndexPage(const TString& relativePath); - + void SortPages(); void ClearPages(); }; - + } diff --git a/library/cpp/monlib/service/pages/mon_page.cpp b/library/cpp/monlib/service/pages/mon_page.cpp index d5860f56b3..72033b1699 100644 --- a/library/cpp/monlib/service/pages/mon_page.cpp +++ b/library/cpp/monlib/service/pages/mon_page.cpp @@ -1,14 +1,14 @@ -#include "mon_page.h" - -using namespace NMonitoring; - +#include "mon_page.h" + +using namespace NMonitoring; + IMonPage::IMonPage(const TString& path, const TString& title) - : Path(path) - , Title(title) -{ + : Path(path) + , Title(title) +{ Y_VERIFY(!Path.StartsWith('/')); Y_VERIFY(!Path.EndsWith('/')); -} +} void IMonPage::OutputNavBar(IOutputStream& out) { TVector<const IMonPage*> parents; diff --git a/library/cpp/monlib/service/pages/mon_page.h b/library/cpp/monlib/service/pages/mon_page.h index 3d85131536..e396612bb0 100644 --- a/library/cpp/monlib/service/pages/mon_page.h +++ b/library/cpp/monlib/service/pages/mon_page.h @@ -1,12 +1,12 @@ -#pragma once - +#pragma once + #include <library/cpp/monlib/service/service.h> #include <library/cpp/monlib/service/mon_service_http_request.h> #include <util/generic/string.h> -#include <util/generic/ptr.h> - -namespace NMonitoring { +#include <util/generic/ptr.h> + +namespace NMonitoring { static const char HTTPOKTEXT[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/plain\r\nConnection: Close\r\n\r\n"; static const char HTTPOKBIN[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/octet-stream\r\nConnection: Close\r\n\r\n"; static const char HTTPOKHTML[] = "HTTP/1.1 200 Ok\r\nContent-Type: text/html\r\nConnection: Close\r\n\r\n"; @@ -19,48 +19,48 @@ namespace NMonitoring { static const char HTTPNOTFOUND[] = "HTTP/1.1 404 Invalid URI\r\nConnection: Close\r\n\r\nInvalid URL\r\n"; static const char HTTPUNAUTHORIZED[] = "HTTP/1.1 401 Unauthorized\r\nConnection: Close\r\n\r\nUnauthorized\r\n"; static const char HTTPFORBIDDEN[] = "HTTP/1.1 403 Forbidden\r\nConnection: Close\r\n\r\nForbidden\r\n"; - + // Fonts static const char HTTPOKFONTEOT[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/vnd.ms-fontobject\r\nConnection: Close\r\n\r\n"; static const char HTTPOKFONTTTF[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/x-font-ttf\r\nConnection: Close\r\n\r\n"; static const char HTTPOKFONTWOFF[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/font-woff\r\nConnection: Close\r\n\r\n"; static const char HTTPOKFONTWOFF2[] = "HTTP/1.1 200 Ok\r\nContent-Type: application/font-woff2\r\nConnection: Close\r\n\r\n"; - + // Images static const char HTTPOKPNG[] = "HTTP/1.1 200 Ok\r\nContent-Type: image/png\r\nConnection: Close\r\n\r\n"; static const char HTTPOKSVG[] = "HTTP/1.1 200 Ok\r\nContent-Type: image/svg+xml\r\nConnection: Close\r\n\r\n"; class IMonPage; - + using TMonPagePtr = TIntrusivePtr<IMonPage>; - + class IMonPage: public TAtomicRefCount<IMonPage> { public: const TString Path; const TString Title; const IMonPage* Parent = nullptr; - + public: IMonPage(const TString& path, const TString& title = TString()); - + virtual ~IMonPage() { } - + void OutputNavBar(IOutputStream& out); virtual const TString& GetPath() const { return Path; } - + virtual const TString& GetTitle() const { return Title; } - + bool IsInIndex() const { return !Title.empty(); } - + virtual void Output(IMonHttpRequest& request) = 0; }; - -} + +} diff --git a/library/cpp/monlib/service/pages/pre_mon_page.cpp b/library/cpp/monlib/service/pages/pre_mon_page.cpp index c2244d559a..fc03a19b80 100644 --- a/library/cpp/monlib/service/pages/pre_mon_page.cpp +++ b/library/cpp/monlib/service/pages/pre_mon_page.cpp @@ -1,18 +1,18 @@ -#include "pre_mon_page.h" - -using namespace NMonitoring; - +#include "pre_mon_page.h" + +using namespace NMonitoring; + void TPreMonPage::OutputContent(NMonitoring::IMonHttpRequest& request) { auto& out = request.Output(); if (PreTag) { - BeforePre(request); + BeforePre(request); out << "<pre>\n"; OutputText(out, request); out << "</pre>\n"; } else { OutputText(out, request); } -} - +} + void TPreMonPage::BeforePre(NMonitoring::IMonHttpRequest&) { -} +} diff --git a/library/cpp/monlib/service/pages/pre_mon_page.h b/library/cpp/monlib/service/pages/pre_mon_page.h index e9cc871d8c..c9a923d39a 100644 --- a/library/cpp/monlib/service/pages/pre_mon_page.h +++ b/library/cpp/monlib/service/pages/pre_mon_page.h @@ -1,8 +1,8 @@ -#pragma once - -#include "html_mon_page.h" - -namespace NMonitoring { +#pragma once + +#include "html_mon_page.h" + +namespace NMonitoring { struct TPreMonPage: public THtmlMonPage { TPreMonPage(const TString& path, const TString& title = TString(), @@ -12,15 +12,15 @@ namespace NMonitoring { , PreTag(preTag) { } - + void OutputContent(NMonitoring::IMonHttpRequest& request) override; - + // hook to customize output virtual void BeforePre(NMonitoring::IMonHttpRequest& request); - + // put your text here virtual void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) = 0; - + const bool PreTag; }; diff --git a/library/cpp/monlib/service/pages/templates.cpp b/library/cpp/monlib/service/pages/templates.cpp index a155b3bab4..ece12bea71 100644 --- a/library/cpp/monlib/service/pages/templates.cpp +++ b/library/cpp/monlib/service/pages/templates.cpp @@ -1,9 +1,9 @@ #include "templates.h" namespace NMonitoring { - extern const char HtmlTag[] = "html"; - extern const char HeadTag[] = "head"; - extern const char BodyTag[] = "body"; + extern const char HtmlTag[] = "html"; + extern const char HeadTag[] = "head"; + extern const char BodyTag[] = "body"; extern const char DivTag[] = "div"; extern const char TableTag[] = "table"; extern const char TableHeadTag[] = "thead"; @@ -15,14 +15,14 @@ namespace NMonitoring { extern const char LabelTag[] = "label"; extern const char SpanTag[] = "span"; extern const char CaptionTag[] = "caption"; - extern const char PreTag[] = "pre"; + extern const char PreTag[] = "pre"; extern const char ParaTag[] = "p"; - extern const char H1Tag[] = "h1"; - extern const char H2Tag[] = "h2"; - extern const char H3Tag[] = "h3"; - extern const char H4Tag[] = "h4"; - extern const char H5Tag[] = "h5"; - extern const char H6Tag[] = "h6"; + extern const char H1Tag[] = "h1"; + extern const char H2Tag[] = "h2"; + extern const char H3Tag[] = "h3"; + extern const char H4Tag[] = "h4"; + extern const char H5Tag[] = "h5"; + extern const char H6Tag[] = "h6"; extern const char SmallTag[] = "small"; extern const char StrongTag[] = "strong"; extern const char ListTag[] = "li"; diff --git a/library/cpp/monlib/service/pages/templates.h b/library/cpp/monlib/service/pages/templates.h index 0c70fb8555..b4656f059f 100644 --- a/library/cpp/monlib/service/pages/templates.h +++ b/library/cpp/monlib/service/pages/templates.h @@ -19,7 +19,7 @@ #define TAG_CLASS_ID(name, cls, id) WITH_SCOPED(tmp, NMonitoring::name(__stream, cls, "", id)) #define TAG_CLASS_FOR(name, cls, for0) WITH_SCOPED(tmp, NMonitoring::name(__stream, cls, for0)) #define TAG_ATTRS(name, ...) WITH_SCOPED(tmp, NMonitoring::name(__stream, ##__VA_ARGS__)) - + #define HTML(str) WITH_SCOPED(__stream, NMonitoring::TOutputStreamRef(str)) #define HEAD() TAG(THead) @@ -54,7 +54,7 @@ #define PARA() TAG(TPara) #define PARA_CLASS(cls) TAG_CLASS(TPara, cls) - + #define H1() TAG(TH1) #define H1_CLASS(cls) TAG_CLASS(TH1, cls) #define H2() TAG(TH2) @@ -235,9 +235,9 @@ namespace NMonitoring { extern const char DTermTag[3]; extern const char DDescTag[3]; - typedef TTag<HtmlTag> THtml; - typedef TTag<HeadTag> THead; - typedef TTag<BodyTag> TBody; + typedef TTag<HtmlTag> THtml; + typedef TTag<HeadTag> THead; + typedef TTag<BodyTag> TBody; typedef TTag<DivTag> TDiv; typedef TTag<TableTag> TTable; typedef TTag<TableHeadTag> TTableHead; @@ -249,14 +249,14 @@ namespace NMonitoring { typedef TTag<LabelTag> TLabelC; typedef TTag<SpanTag> TSpanC; typedef TTag<CaptionTag> TCaption; - typedef TTag<PreTag> TPre; + typedef TTag<PreTag> TPre; typedef TTag<ParaTag> TPara; - typedef TTag<H1Tag> TH1; - typedef TTag<H2Tag> TH2; - typedef TTag<H3Tag> TH3; - typedef TTag<H4Tag> TH4; - typedef TTag<H5Tag> TH5; - typedef TTag<H6Tag> TH6; + typedef TTag<H1Tag> TH1; + typedef TTag<H2Tag> TH2; + typedef TTag<H3Tag> TH3; + typedef TTag<H4Tag> TH4; + typedef TTag<H5Tag> TH5; + typedef TTag<H6Tag> TH6; typedef TTag<SmallTag> TSMALL; typedef TTag<StrongTag> TSTRONG; typedef TTag<ListTag> TLIST; diff --git a/library/cpp/monlib/service/pages/version_mon_page.cpp b/library/cpp/monlib/service/pages/version_mon_page.cpp index 0afc042cc1..41e29417da 100644 --- a/library/cpp/monlib/service/pages/version_mon_page.cpp +++ b/library/cpp/monlib/service/pages/version_mon_page.cpp @@ -1,16 +1,16 @@ #include <library/cpp/svnversion/svnversion.h> #include <library/cpp/build_info/build_info.h> #include <library/cpp/malloc/api/malloc.h> - -#include "version_mon_page.h" - -using namespace NMonitoring; - + +#include "version_mon_page.h" + +using namespace NMonitoring; + void TVersionMonPage::OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) { - const char* version = GetProgramSvnVersion(); - out << version; + const char* version = GetProgramSvnVersion(); + out << version; if (!TString(version).EndsWith("\n")) - out << "\n"; + out << "\n"; out << GetBuildInfo() << "\n\n"; - out << "linked with malloc: " << NMalloc::MallocInfo().Name << "\n"; -} + out << "linked with malloc: " << NMalloc::MallocInfo().Name << "\n"; +} diff --git a/library/cpp/monlib/service/pages/version_mon_page.h b/library/cpp/monlib/service/pages/version_mon_page.h index 2b342c679a..f7649947e4 100644 --- a/library/cpp/monlib/service/pages/version_mon_page.h +++ b/library/cpp/monlib/service/pages/version_mon_page.h @@ -1,15 +1,15 @@ -#pragma once - -#include "pre_mon_page.h" - -namespace NMonitoring { +#pragma once + +#include "pre_mon_page.h" + +namespace NMonitoring { struct TVersionMonPage: public TPreMonPage { TVersionMonPage(const TString& path = "ver", const TString& title = "Version") : TPreMonPage(path, title) { } - + void OutputText(IOutputStream& out, NMonitoring::IMonHttpRequest&) override; }; - + } diff --git a/library/cpp/monlib/ya.make b/library/cpp/monlib/ya.make index b0cd3d0149..9bd236d6fd 100644 --- a/library/cpp/monlib/ya.make +++ b/library/cpp/monlib/ya.make @@ -2,7 +2,7 @@ OWNER( g:solomon jamel ) - + RECURSE( consumers counters diff --git a/library/cpp/on_disk/chunks/chunked_helpers.cpp b/library/cpp/on_disk/chunks/chunked_helpers.cpp index 5d57b12050..b7adba2753 100644 --- a/library/cpp/on_disk/chunks/chunked_helpers.cpp +++ b/library/cpp/on_disk/chunks/chunked_helpers.cpp @@ -4,7 +4,7 @@ TBlob GetBlock(const TBlob& blob, size_t index) { TChunkedDataReader reader(blob); - if (index >= reader.GetBlocksCount()) + if (index >= reader.GetBlocksCount()) ythrow yexception() << "index " << index << " is >= than block count " << reader.GetBlocksCount(); size_t begin = (const char*)reader.GetBlock(index) - (const char*)blob.Data(); return blob.SubBlob(begin, begin + reader.GetBlockLen(index)); diff --git a/library/cpp/packers/packers.h b/library/cpp/packers/packers.h index f8dbb72777..1bde1b59aa 100644 --- a/library/cpp/packers/packers.h +++ b/library/cpp/packers/packers.h @@ -21,11 +21,11 @@ public: void PackLeaf(char*, const T&, size_t) const { } - size_t MeasureLeaf(const T&) const { + size_t MeasureLeaf(const T&) const { return 0; } - size_t SkipLeaf(const char*) const { + size_t SkipLeaf(const char*) const { return 0; } }; @@ -98,13 +98,13 @@ namespace NPackers { class TIntegralPacker { // can pack only integral types <= ui64 public: void UnpackLeaf(const char* p, T& t) const; - void PackLeaf(char* buffer, const T& data, size_t size) const; - size_t MeasureLeaf(const T& data) const; - size_t SkipLeaf(const char* p) const; + void PackLeaf(char* buffer, const T& data, size_t size) const; + size_t MeasureLeaf(const T& data) const; + size_t SkipLeaf(const char* p) const; }; template <> - inline size_t TIntegralPacker<ui64>::MeasureLeaf(const ui64& val) const { + inline size_t TIntegralPacker<ui64>::MeasureLeaf(const ui64& val) const { constexpr size_t MAX_SIZE = sizeof(ui64) + sizeof(ui64) / 8; ui64 value = val; @@ -118,7 +118,7 @@ namespace NPackers { } template <> - inline void TIntegralPacker<ui64>::PackLeaf(char* buffer, const ui64& val, size_t len) const { + inline void TIntegralPacker<ui64>::PackLeaf(char* buffer, const ui64& val, size_t len) const { ui64 value = val; int lenmask = 0; @@ -145,7 +145,7 @@ namespace NPackers { } template <> - inline size_t TIntegralPacker<ui64>::SkipLeaf(const char* p) const { + inline size_t TIntegralPacker<ui64>::SkipLeaf(const char* p) const { return SkipTable[(ui8)*p]; } @@ -183,17 +183,17 @@ namespace NPackers { template <class T> inline void TIntegralPacker<T>::PackLeaf(char* buffer, const T& data, size_t size) const { - TIntegralPacker<ui64>().PackLeaf(buffer, ConvertIntegral<T>(data), size); + TIntegralPacker<ui64>().PackLeaf(buffer, ConvertIntegral<T>(data), size); } template <class T> inline size_t TIntegralPacker<T>::MeasureLeaf(const T& data) const { - return TIntegralPacker<ui64>().MeasureLeaf(ConvertIntegral<T>(data)); + return TIntegralPacker<ui64>().MeasureLeaf(ConvertIntegral<T>(data)); } template <class T> inline size_t TIntegralPacker<T>::SkipLeaf(const char* p) const { - return TIntegralPacker<ui64>().SkipLeaf(p); + return TIntegralPacker<ui64>().SkipLeaf(p); } //------------------------------------------- @@ -257,14 +257,14 @@ namespace NPackers { void UnpackLeaf(const char* p, TStringType& t) const; void PackLeaf(char* buffer, const TStringType& data, size_t size) const; size_t MeasureLeaf(const TStringType& data) const; - size_t SkipLeaf(const char* p) const; + size_t SkipLeaf(const char* p) const; }; template <class TStringType> inline void TStringPacker<TStringType>::UnpackLeaf(const char* buf, TStringType& t) const { size_t len; TIntegralPacker<size_t>().UnpackLeaf(buf, len); - size_t start = TIntegralPacker<size_t>().SkipLeaf(buf); + size_t start = TIntegralPacker<size_t>().SkipLeaf(buf); t = TStringType((const typename TStringType::char_type*)(buf + start), len); } @@ -273,7 +273,7 @@ namespace NPackers { size_t len = str.size(); size_t lenChar = len * sizeof(typename TStringType::char_type); size_t start = size - lenChar; - TIntegralPacker<size_t>().PackLeaf(buf, len, TIntegralPacker<size_t>().MeasureLeaf(len)); + TIntegralPacker<size_t>().PackLeaf(buf, len, TIntegralPacker<size_t>().MeasureLeaf(len)); memcpy(buf + start, str.data(), lenChar); } diff --git a/library/cpp/packers/ut/packers_ut.cpp b/library/cpp/packers/ut/packers_ut.cpp index 7c83c34067..18ce2150d1 100644 --- a/library/cpp/packers/ut/packers_ut.cpp +++ b/library/cpp/packers/ut/packers_ut.cpp @@ -11,8 +11,8 @@ #include <util/generic/ptr.h> #include <util/generic/ylimits.h> -#include <util/folder/dirut.h> - +#include <util/folder/dirut.h> + #include <util/random/random.h> #include <util/string/hex.h> diff --git a/library/cpp/protobuf/util/is_equal.cpp b/library/cpp/protobuf/util/is_equal.cpp index 9a0e94b57e..227408006e 100644 --- a/library/cpp/protobuf/util/is_equal.cpp +++ b/library/cpp/protobuf/util/is_equal.cpp @@ -4,8 +4,8 @@ #include <google/protobuf/descriptor.h> #include <util/generic/yexception.h> -#include <util/string/cast.h> -#include <util/string/vector.h> +#include <util/string/cast.h> +#include <util/string/vector.h> namespace NProtoBuf { template <bool useDefault> @@ -19,7 +19,7 @@ namespace NProtoBuf { return value1 == value2; } }; - + template <bool useDefault> struct TCompareValue<FieldDescriptor::CPPTYPE_MESSAGE, useDefault> { static inline bool IsEqual(const Message* value1, const Message* value2, TVector<TString>* differentPath) { @@ -113,7 +113,7 @@ namespace NProtoBuf { const Descriptor* descr = m1.GetDescriptor(); if (descr != m2.GetDescriptor()) { return false; - } + } for (int i = 0; i < descr->field_count(); ++i) if (!IsEqualField<useDefault>(m1, m2, *descr->field(i), differentPath)) { return false; @@ -148,14 +148,14 @@ namespace NProtoBuf { const Descriptor* descr = m1.GetDescriptor(); if (descr != m2.GetDescriptor()) { return false; - } + } return IsEqualField<useDefault>(m1, m2, field, differentPath); } bool IsEqualField(const Message& m1, const Message& m2, const FieldDescriptor& field) { return IsEqualFieldImpl<false>(m1, m2, field, nullptr); - } - + } + bool IsEqualFieldDefault(const Message& m1, const Message& m2, const FieldDescriptor& field) { return IsEqualFieldImpl<true>(m1, m2, field, nullptr); } diff --git a/library/cpp/protobuf/util/is_equal.h b/library/cpp/protobuf/util/is_equal.h index b351b0e730..13c0aae63d 100644 --- a/library/cpp/protobuf/util/is_equal.h +++ b/library/cpp/protobuf/util/is_equal.h @@ -8,7 +8,7 @@ namespace google { class FieldDescriptor; } } - + namespace NProtoBuf { using ::google::protobuf::FieldDescriptor; using ::google::protobuf::Message; @@ -23,7 +23,7 @@ namespace NProtoBuf { bool IsEqual(const Message& m1, const Message& m2, TString* differentPath); bool IsEqualField(const Message& m1, const Message& m2, const FieldDescriptor& field); - + // Non-strict version: optional field without explicit value is compared // using its default value. bool IsEqualDefault(const Message& m1, const Message& m2); diff --git a/library/cpp/protobuf/util/is_equal_ut.cpp b/library/cpp/protobuf/util/is_equal_ut.cpp index 73c6e41a82..3ca4c90dd5 100644 --- a/library/cpp/protobuf/util/is_equal_ut.cpp +++ b/library/cpp/protobuf/util/is_equal_ut.cpp @@ -1,8 +1,8 @@ -#include "is_equal.h" +#include "is_equal.h" #include <library/cpp/protobuf/util/ut/sample_for_is_equal.pb.h> - + #include <library/cpp/testing/unittest/registar.h> - + #include <google/protobuf/descriptor.h> Y_UNIT_TEST_SUITE(ProtobufIsEqual) { @@ -18,37 +18,37 @@ Y_UNIT_TEST_SUITE(ProtobufIsEqual) { } Y_UNIT_TEST(IsEqual1) { - TSampleForIsEqual a; - TSampleForIsEqual b; - - a.SetName("aaa"); - b.SetName("bbb"); - + TSampleForIsEqual a; + TSampleForIsEqual b; + + a.SetName("aaa"); + b.SetName("bbb"); + TString path; - - bool equal = NProtoBuf::IsEqual(a, b, &path); - UNIT_ASSERT(!equal); - UNIT_ASSERT_VALUES_EQUAL("Name", path); + + bool equal = NProtoBuf::IsEqual(a, b, &path); + UNIT_ASSERT(!equal); + UNIT_ASSERT_VALUES_EQUAL("Name", path); UNIT_ASSERT(!NProtoBuf::IsEqualField(a, b, *NameDescr)); - } - + } + Y_UNIT_TEST(IsEqual2) { - TSampleForIsEqual a; - TSampleForIsEqual b; - - a.MutableInner()->SetBrbrbr("aaa"); - b.MutableInner()->SetBrbrbr("bbb"); - + TSampleForIsEqual a; + TSampleForIsEqual b; + + a.MutableInner()->SetBrbrbr("aaa"); + b.MutableInner()->SetBrbrbr("bbb"); + TString path; - - bool equal = NProtoBuf::IsEqual(a, b, &path); - UNIT_ASSERT(!equal); - UNIT_ASSERT_VALUES_EQUAL("Inner/Brbrbr", path); + + bool equal = NProtoBuf::IsEqual(a, b, &path); + UNIT_ASSERT(!equal); + UNIT_ASSERT_VALUES_EQUAL("Inner/Brbrbr", path); bool equalField = NProtoBuf::IsEqualField(a, b, *InnerDescr); UNIT_ASSERT(!equalField); - } + } Y_UNIT_TEST(IsEqual3) { TSampleForIsEqual a; @@ -85,4 +85,4 @@ Y_UNIT_TEST_SUITE(ProtobufIsEqual) { UNIT_ASSERT(!NProtoBuf::IsEqualField(a, b, *NameDescr)); UNIT_ASSERT(NProtoBuf::IsEqualFieldDefault(a, b, *NameDescr)); } -} +} diff --git a/library/cpp/protobuf/util/repeated_field_utils.h b/library/cpp/protobuf/util/repeated_field_utils.h index 9d53a797d8..c07bd84647 100644 --- a/library/cpp/protobuf/util/repeated_field_utils.h +++ b/library/cpp/protobuf/util/repeated_field_utils.h @@ -1,20 +1,20 @@ -#pragma once - +#pragma once + #include <google/protobuf/repeated_field.h> #include <util/generic/vector.h> - -template <typename T> -void RemoveRepeatedPtrFieldElement(google::protobuf::RepeatedPtrField<T>* repeated, unsigned index) { - google::protobuf::RepeatedPtrField<T> r; + +template <typename T> +void RemoveRepeatedPtrFieldElement(google::protobuf::RepeatedPtrField<T>* repeated, unsigned index) { + google::protobuf::RepeatedPtrField<T> r; Y_ASSERT(index < (unsigned)repeated->size()); for (unsigned i = 0; i < (unsigned)repeated->size(); ++i) { - if (i == index) { - continue; - } - r.Add()->Swap(repeated->Mutable(i)); - } - r.Swap(repeated); -} + if (i == index) { + continue; + } + r.Add()->Swap(repeated->Mutable(i)); + } + r.Swap(repeated); +} namespace NProtoBuf { /// Move item to specified position diff --git a/library/cpp/protobuf/util/ut/sample_for_is_equal.proto b/library/cpp/protobuf/util/ut/sample_for_is_equal.proto index d2257d7f6b..a91c16deaa 100644 --- a/library/cpp/protobuf/util/ut/sample_for_is_equal.proto +++ b/library/cpp/protobuf/util/ut/sample_for_is_equal.proto @@ -1,8 +1,8 @@ -message TInner { - optional string Brbrbr = 3; -} - -message TSampleForIsEqual { - optional string Name = 1; - optional TInner Inner = 5; -} +message TInner { + optional string Brbrbr = 3; +} + +message TSampleForIsEqual { + optional string Name = 1; + optional TInner Inner = 5; +} diff --git a/library/cpp/protobuf/util/ut/ya.make b/library/cpp/protobuf/util/ut/ya.make index be07c51347..701ba9a8c8 100644 --- a/library/cpp/protobuf/util/ut/ya.make +++ b/library/cpp/protobuf/util/ut/ya.make @@ -1,10 +1,10 @@ -OWNER(nga) - +OWNER(nga) + UNITTEST_FOR(library/cpp/protobuf/util) - -SRCS( + +SRCS( extensions.proto - sample_for_is_equal.proto + sample_for_is_equal.proto sample_for_simple_reflection.proto common_ut.proto pb_io_ut.cpp @@ -14,6 +14,6 @@ SRCS( repeated_field_utils_ut.cpp walk_ut.cpp merge_ut.cpp -) - -END() +) + +END() diff --git a/library/cpp/protobuf/util/ya.make b/library/cpp/protobuf/util/ya.make index ed47ab6f2a..b62028af58 100644 --- a/library/cpp/protobuf/util/ya.make +++ b/library/cpp/protobuf/util/ya.make @@ -16,7 +16,7 @@ SRCS( path.cpp pb_io.cpp pb_utils.h - repeated_field_utils.h + repeated_field_utils.h simple_reflection.cpp walk.cpp ) diff --git a/library/cpp/sighandler/async_signals_handler.cpp b/library/cpp/sighandler/async_signals_handler.cpp index ebe1eef9b6..00ce1c18fb 100644 --- a/library/cpp/sighandler/async_signals_handler.cpp +++ b/library/cpp/sighandler/async_signals_handler.cpp @@ -4,11 +4,11 @@ #if !defined(_win_) -#include <errno.h> +#include <errno.h> #include <fcntl.h> -#include <signal.h> +#include <signal.h> #include <string.h> - + #include <unistd.h> #if defined(_linux_) @@ -191,7 +191,7 @@ namespace { TAsyncSignalsHandler* SIGNALS_HANDLER = nullptr; } -void SetAsyncSignalHandler(int signum, TAutoPtr<TEventHandler> handler) { +void SetAsyncSignalHandler(int signum, TAutoPtr<TEventHandler> handler) { static TAtomic lock; if (Y_UNLIKELY(SIGNALS_HANDLER == nullptr)) { @@ -214,12 +214,12 @@ void SetAsyncSignalHandler(int, TAutoPtr<TEventHandler>) { #endif -namespace { +namespace { template <typename TFunc> class TFunctionEventHandler: public TEventHandler { TFunc Func; - public: + public: TFunctionEventHandler(TFunc func) { if (func) Func = func; @@ -230,14 +230,14 @@ namespace { Func(signum); } - return 0; - } - }; -} - -void SetAsyncSignalHandler(int signum, void (*handler)(int)) { + return 0; + } + }; +} + +void SetAsyncSignalHandler(int signum, void (*handler)(int)) { SetAsyncSignalHandler(signum, new TFunctionEventHandler<void (*)(int)>(handler)); -} +} void SetAsyncSignalFunction(int signum, std::function<void(int)> func) { typedef std::function<void(int)> TFunc; diff --git a/library/cpp/sighandler/async_signals_handler.h b/library/cpp/sighandler/async_signals_handler.h index a9f81fcd9d..da36365ace 100644 --- a/library/cpp/sighandler/async_signals_handler.h +++ b/library/cpp/sighandler/async_signals_handler.h @@ -1,14 +1,14 @@ #pragma once -#include <util/generic/ptr.h> +#include <util/generic/ptr.h> #include <functional> - + struct TEventHandler { virtual ~TEventHandler() { } virtual int Handle(int signum) = 0; }; -void SetAsyncSignalHandler(int signum, TAutoPtr<TEventHandler> handler); -void SetAsyncSignalHandler(int signum, void (*handler)(int)); +void SetAsyncSignalHandler(int signum, TAutoPtr<TEventHandler> handler); +void SetAsyncSignalHandler(int signum, void (*handler)(int)); void SetAsyncSignalFunction(int signum, std::function<void(int)> func); diff --git a/library/cpp/streams/lz/lz.h b/library/cpp/streams/lz/lz.h index 063828b1e9..3a2eaad88b 100644 --- a/library/cpp/streams/lz/lz.h +++ b/library/cpp/streams/lz/lz.h @@ -16,7 +16,7 @@ * See http://altdevblogaday.com/2011/04/22/survey-of-fast-compression-algorithms-part-1/ * for some comparisons. */ - + struct TDecompressorError: public yexception { }; diff --git a/library/cpp/string_utils/indent_text/indent_text.cpp b/library/cpp/string_utils/indent_text/indent_text.cpp index 07fc9d4200..09a4f6bca8 100644 --- a/library/cpp/string_utils/indent_text/indent_text.cpp +++ b/library/cpp/string_utils/indent_text/indent_text.cpp @@ -1,25 +1,25 @@ #include "indent_text.h" -#include <util/stream/str.h> - +#include <util/stream/str.h> + TString IndentText(TStringBuf text, TStringBuf indent) { - if (text.empty()) + if (text.empty()) return TString(); - - TStringStream ss; + + TStringStream ss; ss.Reserve(text.size() + 20); - - char pc = 0; - for (size_t i = 0; i < text.size(); ++i) { - if (i == 0 || pc == '\n') - ss << indent; - - char c = text.at(i); - ss << c; - pc = c; - } - if (pc != '\n') - ss << '\n'; - - return ss.Str(); -} + + char pc = 0; + for (size_t i = 0; i < text.size(); ++i) { + if (i == 0 || pc == '\n') + ss << indent; + + char c = text.at(i); + ss << c; + pc = c; + } + if (pc != '\n') + ss << '\n'; + + return ss.Str(); +} diff --git a/library/cpp/string_utils/indent_text/indent_text.h b/library/cpp/string_utils/indent_text/indent_text.h index 9951073e2a..7117d6c0ee 100644 --- a/library/cpp/string_utils/indent_text/indent_text.h +++ b/library/cpp/string_utils/indent_text/indent_text.h @@ -1,6 +1,6 @@ -#pragma once - +#pragma once + #include <util/generic/string.h> -#include <util/generic/strbuf.h> - +#include <util/generic/strbuf.h> + TString IndentText(TStringBuf text, TStringBuf indent = TStringBuf(" ")); diff --git a/library/cpp/string_utils/url/url_ut.cpp b/library/cpp/string_utils/url/url_ut.cpp index ca0ab74948..1588013893 100644 --- a/library/cpp/string_utils/url/url_ut.cpp +++ b/library/cpp/string_utils/url/url_ut.cpp @@ -3,18 +3,18 @@ #include <util/string/cast.h> #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TUtilUrlTest) { Y_UNIT_TEST(TestGetHostAndGetHostAndPort) { - UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHost("ya.ru/bebe")); + UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHost("ya.ru/bebe")); UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHostAndPort("ya.ru/bebe")); - UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHost("ya.ru")); + UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHost("ya.ru")); UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHostAndPort("ya.ru")); - UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHost("ya.ru:8080")); + UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHost("ya.ru:8080")); UNIT_ASSERT_VALUES_EQUAL("ya.ru:8080", GetHostAndPort("ya.ru:8080")); - UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHost("ya.ru/bebe:8080")); + UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHost("ya.ru/bebe:8080")); UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHostAndPort("ya.ru/bebe:8080")); - UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHost("ya.ru:8080/bebe")); + UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHost("ya.ru:8080/bebe")); UNIT_ASSERT_VALUES_EQUAL("ya.ru", GetHost("https://ya.ru:8080/bebe")); UNIT_ASSERT_VALUES_EQUAL("www.ya.ru", GetHost("www.ya.ru:8080/bebe")); UNIT_ASSERT_VALUES_EQUAL("www.ya.ru", GetHost("https://www.ya.ru:8080/bebe")); @@ -25,7 +25,7 @@ Y_UNIT_TEST_SUITE(TUtilUrlTest) { // check simple string UNIT_ASSERT_VALUES_EQUAL("some_blender_url", GetHost("some_blender_url")); UNIT_ASSERT_VALUES_EQUAL("", GetHost("")); - } + } Y_UNIT_TEST(TestGetPathAndQuery) { UNIT_ASSERT_VALUES_EQUAL("/", GetPathAndQuery("ru.wikipedia.org")); @@ -278,4 +278,4 @@ Y_UNIT_TEST_SUITE(TUtilUrlTest) { UNIT_ASSERT_VALUES_EQUAL(false, DoesUrlPathStartWithToken("http://bebe", "bebe")); UNIT_ASSERT_VALUES_EQUAL(false, DoesUrlPathStartWithToken("https://bebe/", "bebe")); } -} +} diff --git a/library/cpp/terminate_handler/sample/exception/main.cpp b/library/cpp/terminate_handler/sample/exception/main.cpp index e764101d0e..35bfae8874 100644 --- a/library/cpp/terminate_handler/sample/exception/main.cpp +++ b/library/cpp/terminate_handler/sample/exception/main.cpp @@ -1,15 +1,15 @@ -#include <util/generic/yexception.h> - - -void Foo(unsigned i = 0) { - if (i >= 10) { - ythrow yexception() << "from Foo()"; - } else { - Foo(i + 1); - } -} - -int main() { - Foo(); - return 0; -} +#include <util/generic/yexception.h> + + +void Foo(unsigned i = 0) { + if (i >= 10) { + ythrow yexception() << "from Foo()"; + } else { + Foo(i + 1); + } +} + +int main() { + Foo(); + return 0; +} diff --git a/library/cpp/terminate_handler/sample/exception/ya.make b/library/cpp/terminate_handler/sample/exception/ya.make index 077586fe14..958c26f89a 100644 --- a/library/cpp/terminate_handler/sample/exception/ya.make +++ b/library/cpp/terminate_handler/sample/exception/ya.make @@ -1,13 +1,13 @@ PROGRAM(exception_sample) - -OWNER(nga) - -SRCS( - main.cpp -) - -PEERDIR( + +OWNER(nga) + +SRCS( + main.cpp +) + +PEERDIR( library/cpp/terminate_handler -) - -END() +) + +END() diff --git a/library/cpp/terminate_handler/sample/pure-virtual/main.cpp b/library/cpp/terminate_handler/sample/pure-virtual/main.cpp index 6e9c9fbff4..58217d5f24 100644 --- a/library/cpp/terminate_handler/sample/pure-virtual/main.cpp +++ b/library/cpp/terminate_handler/sample/pure-virtual/main.cpp @@ -1,22 +1,22 @@ - -struct TFoo { - TFoo() { - Baz(); - } - - void Baz() { - Bar(); - } - - virtual void Bar() = 0; -}; - -struct TQux: public TFoo { + +struct TFoo { + TFoo() { + Baz(); + } + + void Baz() { + Bar(); + } + + virtual void Bar() = 0; +}; + +struct TQux: public TFoo { void Bar() override { } -}; - -int main() { - TQux(); - return 0; -} +}; + +int main() { + TQux(); + return 0; +} diff --git a/library/cpp/terminate_handler/sample/pure-virtual/ya.make b/library/cpp/terminate_handler/sample/pure-virtual/ya.make index 27ef18f3a0..4100da630d 100644 --- a/library/cpp/terminate_handler/sample/pure-virtual/ya.make +++ b/library/cpp/terminate_handler/sample/pure-virtual/ya.make @@ -1,13 +1,13 @@ -PROGRAM() - -OWNER(nga) - -SRCS( - main.cpp -) - -PEERDIR( +PROGRAM() + +OWNER(nga) + +SRCS( + main.cpp +) + +PEERDIR( library/cpp/terminate_handler -) - -END() +) + +END() diff --git a/library/cpp/terminate_handler/sample/rethrow/main.cpp b/library/cpp/terminate_handler/sample/rethrow/main.cpp index 57e0722a3d..fcd8592613 100644 --- a/library/cpp/terminate_handler/sample/rethrow/main.cpp +++ b/library/cpp/terminate_handler/sample/rethrow/main.cpp @@ -1,20 +1,20 @@ -#include <util/generic/yexception.h> - - -void Bar() { - ythrow yexception() << "from Foo()"; -} - -void Foo() { - try { - Bar(); - } catch (...) { - Cerr << "caught; rethrowing\n"; - throw; - } -} - -int main() { - Foo(); - return 0; -} +#include <util/generic/yexception.h> + + +void Bar() { + ythrow yexception() << "from Foo()"; +} + +void Foo() { + try { + Bar(); + } catch (...) { + Cerr << "caught; rethrowing\n"; + throw; + } +} + +int main() { + Foo(); + return 0; +} diff --git a/library/cpp/terminate_handler/sample/rethrow/ya.make b/library/cpp/terminate_handler/sample/rethrow/ya.make index 27ef18f3a0..4100da630d 100644 --- a/library/cpp/terminate_handler/sample/rethrow/ya.make +++ b/library/cpp/terminate_handler/sample/rethrow/ya.make @@ -1,13 +1,13 @@ -PROGRAM() - -OWNER(nga) - -SRCS( - main.cpp -) - -PEERDIR( +PROGRAM() + +OWNER(nga) + +SRCS( + main.cpp +) + +PEERDIR( library/cpp/terminate_handler -) - -END() +) + +END() diff --git a/library/cpp/terminate_handler/sample/segv/main.cpp b/library/cpp/terminate_handler/sample/segv/main.cpp index 0b29cc193e..52851bdb19 100644 --- a/library/cpp/terminate_handler/sample/segv/main.cpp +++ b/library/cpp/terminate_handler/sample/segv/main.cpp @@ -1,15 +1,15 @@ -#include "../../segv_handler.h" - -void Bar(int* x) { - *x = 11; -} - -void Foo(int* x) { - Bar(x); -} - -int main() { - InstallSegvHandler(); +#include "../../segv_handler.h" + +void Bar(int* x) { + *x = 11; +} + +void Foo(int* x) { + Bar(x); +} + +int main() { + InstallSegvHandler(); Foo((int*)1); - return 0; -} + return 0; +} diff --git a/library/cpp/terminate_handler/sample/segv/ya.make b/library/cpp/terminate_handler/sample/segv/ya.make index 27ef18f3a0..4100da630d 100644 --- a/library/cpp/terminate_handler/sample/segv/ya.make +++ b/library/cpp/terminate_handler/sample/segv/ya.make @@ -1,13 +1,13 @@ -PROGRAM() - -OWNER(nga) - -SRCS( - main.cpp -) - -PEERDIR( +PROGRAM() + +OWNER(nga) + +SRCS( + main.cpp +) + +PEERDIR( library/cpp/terminate_handler -) - -END() +) + +END() diff --git a/library/cpp/terminate_handler/sample/ya.make b/library/cpp/terminate_handler/sample/ya.make index f60f58e799..af089abc65 100644 --- a/library/cpp/terminate_handler/sample/ya.make +++ b/library/cpp/terminate_handler/sample/ya.make @@ -1,6 +1,6 @@ -RECURSE( +RECURSE( exception pure-virtual rethrow segv -) +) diff --git a/library/cpp/terminate_handler/segv_handler.cpp b/library/cpp/terminate_handler/segv_handler.cpp index fc720eff18..f24ece4125 100644 --- a/library/cpp/terminate_handler/segv_handler.cpp +++ b/library/cpp/terminate_handler/segv_handler.cpp @@ -1,34 +1,34 @@ -#include <util/system/platform.h> -#include <util/system/yassert.h> -#include <util/stream/output.h> -#include <util/system/backtrace.h> - -#ifdef _unix_ -#include <signal.h> -#include <errno.h> -#include <stdlib.h> +#include <util/system/platform.h> +#include <util/system/yassert.h> +#include <util/stream/output.h> +#include <util/system/backtrace.h> + +#ifdef _unix_ +#include <signal.h> +#include <errno.h> +#include <stdlib.h> #include <unistd.h> -#endif - -#include "segv_handler.h" - -#ifndef _win_ -static void SegvHandler(int sig) { +#endif + +#include "segv_handler.h" + +#ifndef _win_ +static void SegvHandler(int sig) { Y_UNUSED(sig); - const char msg[] = "Got SEGV\n"; + const char msg[] = "Got SEGV\n"; Y_UNUSED(write(STDERR_FILENO, msg, sizeof(msg))); - //PrintBackTrace(); - sig_t r = signal(SIGSEGV, SIG_DFL); - if (r == SIG_ERR) { - abort(); - } - // returning back and failing -} -#endif // !_win_ - -void InstallSegvHandler() { -#ifndef _win_ - sig_t r = signal(SIGSEGV, &SegvHandler); + //PrintBackTrace(); + sig_t r = signal(SIGSEGV, SIG_DFL); + if (r == SIG_ERR) { + abort(); + } + // returning back and failing +} +#endif // !_win_ + +void InstallSegvHandler() { +#ifndef _win_ + sig_t r = signal(SIGSEGV, &SegvHandler); Y_VERIFY(r != SIG_ERR, "signal failed: %s", strerror(errno)); -#endif // !_win_ -} +#endif // !_win_ +} diff --git a/library/cpp/terminate_handler/segv_handler.h b/library/cpp/terminate_handler/segv_handler.h index 5f53810460..c9f9051c7d 100644 --- a/library/cpp/terminate_handler/segv_handler.h +++ b/library/cpp/terminate_handler/segv_handler.h @@ -1,3 +1,3 @@ -#pragma once - -void InstallSegvHandler(); +#pragma once + +void InstallSegvHandler(); diff --git a/library/cpp/terminate_handler/terminate_handler.cpp b/library/cpp/terminate_handler/terminate_handler.cpp index 30f17ae4bf..d7e8fbed95 100644 --- a/library/cpp/terminate_handler/terminate_handler.cpp +++ b/library/cpp/terminate_handler/terminate_handler.cpp @@ -1,15 +1,15 @@ #include <cstdlib> -#include <exception> - -#include <util/stream/output.h> -#include <util/system/backtrace.h> -#include <util/generic/yexception.h> - +#include <exception> + +#include <util/stream/output.h> +#include <util/system/backtrace.h> +#include <util/generic/yexception.h> + namespace { // Avoid infinite recursion if std::terminate is triggered anew by the // FancyTerminateHandler. thread_local int TerminateCount = 0; - + void FancyTerminateHandler() { switch (++TerminateCount) { case 1: @@ -21,7 +21,7 @@ namespace { abort(); break; } - + if (std::current_exception()) { Cerr << "Uncaught exception: " << CurrentExceptionMessage() << '\n'; } else { @@ -29,8 +29,8 @@ namespace { } PrintBackTrace(); Cerr.Flush(); - abort(); - } - + abort(); + } + [[maybe_unused]] auto _ = std::set_terminate(&FancyTerminateHandler); -} +} diff --git a/library/cpp/terminate_handler/ya.make b/library/cpp/terminate_handler/ya.make index 40ca71a5cc..70a9712fed 100644 --- a/library/cpp/terminate_handler/ya.make +++ b/library/cpp/terminate_handler/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - +LIBRARY() + OWNER( ilnurkh eeight ) - -SRCS( + +SRCS( GLOBAL terminate_handler.cpp - segv_handler.cpp -) - -END() + segv_handler.cpp +) + +END() diff --git a/library/cpp/testing/unittest/registar.cpp b/library/cpp/testing/unittest/registar.cpp index bfb87970d1..3679b768ed 100644 --- a/library/cpp/testing/unittest/registar.cpp +++ b/library/cpp/testing/unittest/registar.cpp @@ -5,10 +5,10 @@ #include <util/generic/bt_exception.h> #include <util/random/fast.h> -#include <util/string/printf.h> +#include <util/string/printf.h> #include <util/system/backtrace.h> #include <util/system/guard.h> -#include <util/system/tls.h> +#include <util/system/tls.h> #include <util/system/error.h> #include <util/string/cast.h> @@ -27,13 +27,13 @@ TString NUnitTest::RandomString(size_t len, ui32 seed) { return ret; } - + Y_POD_STATIC_THREAD(bool) UnittestThread; Y_POD_STATIC_THREAD(NUnitTest::TTestBase*) currentTest; ::NUnitTest::TRaiseErrorHandler RaiseErrorHandler; - + void ::NUnitTest::NPrivate::RaiseError(const char* what, const TString& msg, bool fatalFailure) { Y_VERIFY(UnittestThread, "%s in non-unittest thread with message:\n%s", what, msg.data()); Y_VERIFY(GetCurrentTest()); @@ -51,17 +51,17 @@ void ::NUnitTest::NPrivate::RaiseError(const char* what, const TString& msg, boo return; } throw TAssertException(); -} - +} + void ::NUnitTest::SetRaiseErrorHandler(::NUnitTest::TRaiseErrorHandler handler) { Y_VERIFY(UnittestThread); RaiseErrorHandler = std::move(handler); } -void ::NUnitTest::NPrivate::SetUnittestThread(bool unittestThread) { +void ::NUnitTest::NPrivate::SetUnittestThread(bool unittestThread) { Y_VERIFY(UnittestThread != unittestThread, "state check"); - UnittestThread = unittestThread; -} + UnittestThread = unittestThread; +} void ::NUnitTest::NPrivate::SetCurrentTest(TTestBase* test) { Y_VERIFY(!test || !currentTest, "state check"); diff --git a/library/cpp/testing/unittest/registar.h b/library/cpp/testing/unittest/registar.h index 594e6681e7..44517a0092 100644 --- a/library/cpp/testing/unittest/registar.h +++ b/library/cpp/testing/unittest/registar.h @@ -38,7 +38,7 @@ namespace NUnitTest { void SetCurrentTest(TTestBase*); TTestBase* GetCurrentTest(); } - + extern bool ShouldColorizeDiff; extern bool ContinueOnFail; TString ColoredDiff(TStringBuf s1, TStringBuf s2, const TString& delims = TString(), bool reverse = false); @@ -122,19 +122,19 @@ namespace NUnitTest { void Finish(const TFinish& descr); unsigned GoodTests() const noexcept; - + unsigned FailTests() const noexcept; unsigned GoodTestsInCurrentUnit() const noexcept; unsigned FailTestsInCurrentUnit() const noexcept; - // Should execute test suite? + // Should execute test suite? virtual bool CheckAccess(TString /*name*/, size_t /*num*/); - // Should execute a test whitin suite? + // Should execute a test whitin suite? virtual bool CheckAccessTest(TString /*suite*/, const char* /*name*/); - + virtual void Run(std::function<void()> f, const TString& /*suite*/, const char* /*name*/, bool /*forceFork*/); // This process is forked for current test @@ -157,7 +157,7 @@ namespace NUnitTest { virtual void OnFinish(const TFinish* /*finish*/); virtual void OnBeforeTest(const TTest* /*test*/); - + void AddTestError(const TTest& test); void AddTestFinish(const TTest& test); @@ -176,7 +176,7 @@ namespace NUnitTest { virtual ~ITestBaseFactory(); - // name of test suite + // name of test suite virtual TString Name() const noexcept = 0; virtual TTestBase* ConstructTest() = 0; @@ -210,9 +210,9 @@ namespace NUnitTest { protected: bool CheckAccessTest(const char* test); - + void BeforeTest(const char* func); - + void Finish(const char* func, TTestContext* context); void AtStart(); @@ -433,7 +433,7 @@ public: \ auto&& failMsg = Sprintf("%s != %s %s", ToString(_a).data(), ToString(_b).data(), (::TStringBuilder() << C).data()); \ UNIT_FAIL_IMPL("strings equal assertion failed", failMsg); \ } \ - } while (false) + } while (false) #define UNIT_ASSERT_STRINGS_EQUAL(A, B) UNIT_ASSERT_STRINGS_EQUAL_C(A, B, "") @@ -702,7 +702,7 @@ public: \ return false; } } - + //values #define UNIT_ASSERT_VALUES_EQUAL_IMPL(A, B, C, EQflag, EQstr, NEQstr) \ do { \ @@ -720,7 +720,7 @@ public: \ UNIT_FAIL_IMPL("assertion failed", failMsg); \ } \ } while (false) - + #define UNIT_ASSERT_VALUES_EQUAL_C(A, B, C) \ UNIT_ASSERT_VALUES_EQUAL_IMPL(A, B, C, true, "==", "!=") diff --git a/library/cpp/testing/unittest/utmain.cpp b/library/cpp/testing/unittest/utmain.cpp index 6cc2bfbbb1..305bc6b40f 100644 --- a/library/cpp/testing/unittest/utmain.cpp +++ b/library/cpp/testing/unittest/utmain.cpp @@ -216,7 +216,7 @@ public: inline void SetPrintBeforeSuite(bool print) { PrintBeforeSuite_ = print; } - + inline void SetPrintAfterSuite(bool print) { PrintAfterSuite_ = print; } @@ -224,7 +224,7 @@ public: inline void SetPrintBeforeTest(bool print) { PrintBeforeTest_ = print; } - + inline void SetPrintAfterTest(bool print) { PrintAfterTest_ = print; } @@ -286,7 +286,7 @@ public: inline void SetLoop(bool loop) { Loop = loop; } - + inline bool IsLoop() const { return Loop; } @@ -300,17 +300,17 @@ private: TraceProcessor->UnitStart(*unit); if (IsForked) { return; - } + } if (PrintBeforeSuite_ || PrintBeforeTest_) { fprintf(stderr, "%s<-----%s %s\n", LightBlueColor().data(), OldColor().data(), unit->name.data()); } } - + void OnUnitStop(const TUnit* unit) override { TraceProcessor->UnitStop(*unit); if (IsForked) { return; - } + } if (!PrintAfterSuite_) { return; } @@ -329,12 +329,12 @@ private: TraceProcessor->BeforeTest(*test); if (IsForked) { return; - } + } if (PrintBeforeTest_) { fprintf(stderr, "[%sexec%s] %s::%s...\n", LightBlueColor().data(), OldColor().data(), test->unit->name.data(), test->name); } } - + void OnError(const TError* descr) override { TraceProcessor->Error(*descr); if (!IsForked && ForkExitedCorrectly) { @@ -454,19 +454,19 @@ private: if (EnabledTests_.empty()) { return true; } - + if (EnabledTests_.find(TString() + suite + "::*") != EnabledTests_.end()) { return true; } - + return EnabledTests_.find(name) != EnabledTests_.end(); } - + void Run(std::function<void()> f, const TString& suite, const char* name, const bool forceFork) override { if (!(ForkTests || forceFork) || GetIsForked()) { return f(); - } - + } + TList<TString> args(1, "--is-forked-internal"); args.push_back(Sprintf("+%s::%s", suite.data(), name)); @@ -614,7 +614,7 @@ static int DoUsage(const char* progname) { << " -l, --list print a list of available tests\n" << " -A --list-verbose print a list of available subtests\n" << " --print-before-test print each test name before running it\n" - << " --print-before-suite print each test suite name before running it\n" + << " --print-before-suite print each test suite name before running it\n" << " --show-fails print a list of all failed tests at the end\n" << " --dont-show-fails do not print a list of all failed tests at the end\n" << " --continue-on-fail print a message and continue running test suite instead of break\n" @@ -686,9 +686,9 @@ int NUnitTest::RunMain(int argc, char** argv) { } else if (strcmp(name, "--list-verbose") == 0 || strcmp(name, "-A") == 0) { listTests = LIST_VERBOSE; } else if (strcmp(name, "--print-before-suite=false") == 0) { - processor.SetPrintBeforeSuite(false); - } else if (strcmp(name, "--print-before-test=false") == 0) { - processor.SetPrintBeforeTest(false); + processor.SetPrintBeforeSuite(false); + } else if (strcmp(name, "--print-before-test=false") == 0) { + processor.SetPrintBeforeTest(false); } else if (strcmp(name, "--print-before-suite") == 0) { processor.SetPrintBeforeSuite(true); } else if (strcmp(name, "--print-before-test") == 0) { @@ -709,10 +709,10 @@ int NUnitTest::RunMain(int argc, char** argv) { processor.SetEnd(FromString<size_t>(argv[i])); } else if (strcmp(name, "--fork-tests") == 0) { processor.SetForkTests(true); - } else if (strcmp(name, "--is-forked-internal") == 0) { + } else if (strcmp(name, "--is-forked-internal") == 0) { processor.SetIsForked(true); - } else if (strcmp(name, "--loop") == 0) { - processor.SetLoop(true); + } else if (strcmp(name, "--loop") == 0) { + processor.SetLoop(true); } else if (strcmp(name, "--trace-path") == 0) { ++i; processor.BeQuiet(); @@ -733,8 +733,8 @@ int NUnitTest::RunMain(int argc, char** argv) { size_t assign = param.find('='); Singleton<::NPrivate::TTestEnv>()->AddTestParam(param.substr(0, assign), param.substr(assign + 1)); } else if (TString(name).StartsWith("--")) { - return DoUsage(argv[0]), 1; - } else if (*name == '-') { + return DoUsage(argv[0]), 1; + } else if (*name == '-') { processor.Disable(name + 1); } else if (*name == '+') { processor.Enable(name + 1); @@ -750,15 +750,15 @@ int NUnitTest::RunMain(int argc, char** argv) { TTestFactory::Instance().SetProcessor(&processor); unsigned ret; - for (;;) { + for (;;) { ret = TTestFactory::Instance().Execute(); if (!processor.GetIsForked() && ret && processor.GetPrintSummary()) { - Cerr << "SOME TESTS FAILED!!!!" << Endl; - } - + Cerr << "SOME TESTS FAILED!!!!" << Endl; + } + if (0 != ret || !processor.IsLoop()) { break; - } + } } return ret; #ifndef UT_SKIP_EXCEPTIONS diff --git a/library/cpp/uri/common.h b/library/cpp/uri/common.h index ebe011731d..8025357763 100644 --- a/library/cpp/uri/common.h +++ b/library/cpp/uri/common.h @@ -487,7 +487,7 @@ namespace NUri { const char* FieldToString(const TField::EField& t); const char* ParsedStateToString(const TState::EParsed& t); const char* SchemeKindToString(const TScheme::EKind& t); - + } Y_DECLARE_OUT_SPEC(inline, NUri::TField::EField, out, t) { diff --git a/library/cpp/uri/uri.h b/library/cpp/uri/uri.h index 8481ff6780..3b6c19fe4a 100644 --- a/library/cpp/uri/uri.h +++ b/library/cpp/uri/uri.h @@ -616,7 +616,7 @@ namespace NUri { const char* LinkTypeToString(const TUri::TLinkType& t); } - + Y_DECLARE_OUT_SPEC(inline, NUri::TUri, out, url) { url.Print(out); } diff --git a/util/charset/wide.h b/util/charset/wide.h index 4c681679ee..04e6928aab 100644 --- a/util/charset/wide.h +++ b/util/charset/wide.h @@ -312,7 +312,7 @@ inline size_t UTF8ToWideImpl(const char* text, size_t len, TCharType* dest, size ::NDetail::UTF8ToWideImplScalar<robust>(cur, last, p); written = p - dest; - return cur - reinterpret_cast<const unsigned char*>(text); + return cur - reinterpret_cast<const unsigned char*>(text); } template <typename TCharType> @@ -337,15 +337,15 @@ inline bool UTF8ToWide(const char* text, size_t len, TCharType* dest, size_t& wr return UTF8ToWideImpl<robust>(text, len, dest, written) == len; } -//! converts text from UTF8 to unicode, stops immediately it UTF8 byte sequence is wrong -//! @attention destination buffer must be long enough to fit all characters of the text, -//! conversion stops if a broken symbol is met -//! @return @c true if all the text converted successfully, @c false - a broken symbol was found +//! converts text from UTF8 to unicode, stops immediately it UTF8 byte sequence is wrong +//! @attention destination buffer must be long enough to fit all characters of the text, +//! conversion stops if a broken symbol is met +//! @return @c true if all the text converted successfully, @c false - a broken symbol was found template <typename TCharType> inline bool UTF8ToWide(const char* text, size_t len, TCharType* dest, size_t& written) noexcept { return UTF8ToWide<false>(text, len, dest, written); -} - +} + template <bool robust> inline TWtringBuf UTF8ToWide(const TStringBuf src, TUtf16String& dst) { dst.ReserveAndResize(src.size()); diff --git a/util/datetime/base.cpp b/util/datetime/base.cpp index f61b7b889f..38ecc3ab96 100644 --- a/util/datetime/base.cpp +++ b/util/datetime/base.cpp @@ -1,26 +1,26 @@ #include "base.h" - -#include <util/string/cast.h> + +#include <util/string/cast.h> #include <util/stream/output.h> #include <util/stream/mem.h> -#include <util/system/compat.h> +#include <util/system/compat.h> #include <util/memory/tempbuf.h> #include <util/generic/string.h> #include <util/generic/strbuf.h> #include <util/generic/yexception.h> - + TString Strftime(const char* format, const struct tm* tm) { size_t size = Max<size_t>(strlen(format) * 2 + 1, 107); - for (;;) { + for (;;) { TTempBuf buf(size); - int r = strftime(buf.Data(), buf.Size(), format, tm); + int r = strftime(buf.Data(), buf.Size(), format, tm); if (r != 0) { return TString(buf.Data(), r); } - size *= 2; - } -} - + size *= 2; + } +} + template <> TDuration FromStringImpl<TDuration, char>(const char* s, size_t len) { return TDuration::Parse(TStringBuf(s, len)); diff --git a/util/datetime/base.h b/util/datetime/base.h index 152fa983bb..5e902b8f63 100644 --- a/util/datetime/base.h +++ b/util/datetime/base.h @@ -3,14 +3,14 @@ #include "systime.h" #include <util/str_stl.h> -#include <util/system/platform.h> -#include <util/system/datetime.h> +#include <util/system/platform.h> +#include <util/system/datetime.h> #include <util/generic/string.h> #include <util/generic/strbuf.h> -#include <util/generic/ylimits.h> +#include <util/generic/ylimits.h> #include <util/generic/utility.h> #include <util/generic/typetraits.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <chrono> @@ -29,20 +29,20 @@ #pragma warning(disable : 4244) // conversion from 'time_t' to 'long', possible loss of data #endif // _MSC_VER -// Microseconds since epoch -class TInstant; +// Microseconds since epoch +class TInstant; -// Duration is microseconds. Could be used to store timeouts, for example. -class TDuration; - -/// Current time +// Duration is microseconds. Could be used to store timeouts, for example. +class TDuration; + +/// Current time static inline TInstant Now() noexcept; - -/// Use Now() method to obtain current time instead of *Seconds() unless you understand what are you doing. - -class TDateTimeParseException: public yexception { -}; - + +/// Use Now() method to obtain current time instead of *Seconds() unless you understand what are you doing. + +class TDateTimeParseException: public yexception { +}; + const int DATE_BUF_LEN = 4 + 2 + 2 + 1; // [YYYYMMDD*] constexpr long seconds(const struct tm& theTm) { @@ -73,8 +73,8 @@ bool ParseISO8601DateTime(const char* date, time_t& utcTime); bool ParseISO8601DateTime(const char* date, size_t dateLen, time_t& utcTime); bool ParseRFC822DateTime(const char* date, time_t& utcTime); bool ParseRFC822DateTime(const char* date, size_t dateLen, time_t& utcTime); -bool ParseHTTPDateTime(const char* date, time_t& utcTime); -bool ParseHTTPDateTime(const char* date, size_t dateLen, time_t& utcTime); +bool ParseHTTPDateTime(const char* date, time_t& utcTime); +bool ParseHTTPDateTime(const char* date, size_t dateLen, time_t& utcTime); bool ParseX509ValidityDateTime(const char* date, time_t& utcTime); bool ParseX509ValidityDateTime(const char* date, size_t dateLen, time_t& utcTime); @@ -83,7 +83,7 @@ constexpr long TVdiff(timeval r1, timeval r2) { } TString Strftime(const char* format, const struct tm* tm); - + // Use functions below instead of sprint_date (check IGNIETFERRO-892 for details) void DateToString(char* buf, const struct tm& theTm); void DateToString(char* buf, time_t when, long* sec = nullptr); @@ -93,17 +93,17 @@ TString DateToString(time_t when, long* sec = nullptr); TString YearToString(const struct tm& theTm); TString YearToString(time_t when); -template <class S> -class TTimeBase { -public: +template <class S> +class TTimeBase { +public: using TValue = ui64; - + protected: constexpr TTimeBase(const TValue& value) noexcept : Value_(value) - { - } - + { + } + public: constexpr TTimeBase() noexcept : Value_(0) @@ -112,40 +112,40 @@ public: constexpr TTimeBase(const struct timeval& tv) noexcept : Value_(tv.tv_sec * (ui64)1000000 + tv.tv_usec) - { - } - + { + } + constexpr TValue GetValue() const noexcept { return Value_; - } - + } + constexpr double SecondsFloat() const noexcept { return Value_ * (1 / 1000000.0); - } - + } + constexpr double MillisecondsFloat() const noexcept { return Value_ * (1 / 1000.0); } constexpr TValue MicroSeconds() const noexcept { return Value_; - } + } constexpr TValue MilliSeconds() const noexcept { - return MicroSeconds() / 1000; - } + return MicroSeconds() / 1000; + } constexpr TValue Seconds() const noexcept { - return MilliSeconds() / 1000; - } + return MilliSeconds() / 1000; + } constexpr TValue Minutes() const noexcept { - return Seconds() / 60; - } + return Seconds() / 60; + } constexpr TValue Hours() const noexcept { - return Minutes() / 60; - } + return Minutes() / 60; + } constexpr TValue Days() const noexcept { return Hours() / 24; @@ -153,16 +153,16 @@ public: constexpr TValue NanoSeconds() const noexcept { return MicroSeconds() >= (Max<TValue>() / (TValue)1000) ? Max<TValue>() : MicroSeconds() * (TValue)1000; - } + } constexpr ui32 MicroSecondsOfSecond() const noexcept { return MicroSeconds() % (TValue)1000000; - } + } constexpr ui32 MilliSecondsOfSecond() const noexcept { return MicroSecondsOfSecond() / (TValue)1000; - } - + } + constexpr ui32 NanoSecondsOfSecond() const noexcept { return MicroSecondsOfSecond() * (TValue)1000; } @@ -173,26 +173,26 @@ public: protected: TValue Value_; // microseconds count -}; - -namespace NDateTimeHelpers { - template <typename T> - struct TPrecisionHelper { +}; + +namespace NDateTimeHelpers { + template <typename T> + struct TPrecisionHelper { using THighPrecision = ui64; - }; - - template <> - struct TPrecisionHelper<float> { + }; + + template <> + struct TPrecisionHelper<float> { using THighPrecision = double; - }; - - template <> - struct TPrecisionHelper<double> { + }; + + template <> + struct TPrecisionHelper<double> { using THighPrecision = double; - }; + }; } - -class TDuration: public TTimeBase<TDuration> { + +class TDuration: public TTimeBase<TDuration> { using TBase = TTimeBase<TDuration>; private: @@ -201,18 +201,18 @@ private: */ constexpr explicit TDuration(TValue value) noexcept : TBase(value) - { - } - + { + } + public: constexpr TDuration() noexcept { } constexpr TDuration(const struct timeval& tv) noexcept - : TBase(tv) - { - } - + : TBase(tv) + { + } + /** * TDuration is compatible with std::chrono::duration: * it can be constructed and compared with std::chrono::duration. @@ -261,52 +261,52 @@ public: } static constexpr TDuration MicroSeconds(ui64 us) noexcept { - return TDuration(us); - } - + return TDuration(us); + } + /* noexcept(false) as conversion from T might throw, for example FromString("abc") */ - template <typename T> + template <typename T> static constexpr TDuration MilliSeconds(T ms) noexcept(false) { return MicroSeconds((ui64)(typename NDateTimeHelpers::TPrecisionHelper<T>::THighPrecision(ms) * 1000)); - } - + } + using TBase::Days; - using TBase::Hours; + using TBase::Hours; using TBase::MicroSeconds; using TBase::MilliSeconds; - using TBase::Minutes; - using TBase::Seconds; - - /// DeadLineFromTimeOut - inline TInstant ToDeadLine() const; + using TBase::Minutes; + using TBase::Seconds; + + /// DeadLineFromTimeOut + inline TInstant ToDeadLine() const; constexpr TInstant ToDeadLine(TInstant now) const; - + static constexpr TDuration Max() noexcept { return TDuration(::Max<TValue>()); - } - + } + static constexpr TDuration Zero() noexcept { return TDuration(); - } - + } + /* noexcept(false) as conversion from T might throw, for example FromString("abc") */ - template <typename T> + template <typename T> static constexpr TDuration Seconds(T s) noexcept(false) { - return MilliSeconds(typename NDateTimeHelpers::TPrecisionHelper<T>::THighPrecision(s) * 1000); - } - + return MilliSeconds(typename NDateTimeHelpers::TPrecisionHelper<T>::THighPrecision(s) * 1000); + } + static constexpr TDuration Minutes(ui64 m) noexcept { - return Seconds(m * 60); - } + return Seconds(m * 60); + } static constexpr TDuration Hours(ui64 h) noexcept { - return Minutes(h * 60); - } + return Minutes(h * 60); + } static constexpr TDuration Days(ui64 d) noexcept { - return Hours(d * 24); - } - + return Hours(d * 24); + } + /// parses strings like 10s, 15ms, 15.05s, 20us, or just 25 (s). See parser_ut.cpp for details static TDuration Parse(const TStringBuf input); @@ -333,10 +333,10 @@ public: inline TDuration& operator/=(const T& t) noexcept { return (*this = (*this / t)); } - + TString ToString() const; -}; - +}; + Y_DECLARE_PODTYPE(TDuration); template <> @@ -346,8 +346,8 @@ struct THash<TDuration> { } }; -/// TInstant and TDuration are guaranteed to have same precision -class TInstant: public TTimeBase<TInstant> { +/// TInstant and TDuration are guaranteed to have same precision +class TInstant: public TTimeBase<TInstant> { using TBase = TTimeBase<TInstant>; private: @@ -356,61 +356,61 @@ private: */ constexpr explicit TInstant(TValue value) noexcept : TBase(value) - { - } - + { + } + public: constexpr TInstant() noexcept { } constexpr TInstant(const struct timeval& tv) noexcept - : TBase(tv) - { - } - + : TBase(tv) + { + } + static constexpr TInstant FromValue(TValue value) noexcept { return TInstant(value); } - static inline TInstant Now() { + static inline TInstant Now() { return TInstant::MicroSeconds(::MicroSeconds()); - } - + } + using TBase::Days; - using TBase::Hours; + using TBase::Hours; using TBase::MicroSeconds; using TBase::MilliSeconds; - using TBase::Minutes; - using TBase::Seconds; - + using TBase::Minutes; + using TBase::Seconds; + static constexpr TInstant Max() noexcept { return TInstant(::Max<TValue>()); - } - + } + static constexpr TInstant Zero() noexcept { return TInstant(); - } - - /// us since epoch + } + + /// us since epoch static constexpr TInstant MicroSeconds(ui64 us) noexcept { - return TInstant(us); - } + return TInstant(us); + } - /// ms since epoch + /// ms since epoch static constexpr TInstant MilliSeconds(ui64 ms) noexcept { - return MicroSeconds(ms * 1000); - } + return MicroSeconds(ms * 1000); + } - /// seconds since epoch + /// seconds since epoch static constexpr TInstant Seconds(ui64 s) noexcept { - return MilliSeconds(s * 1000); - } + return MilliSeconds(s * 1000); + } - /// minutes since epoch + /// minutes since epoch static constexpr TInstant Minutes(ui64 m) noexcept { - return Seconds(m * 60); - } - + return Seconds(m * 60); + } + /// hours since epoch static constexpr TInstant Hours(ui64 h) noexcept { return Minutes(h * 60); @@ -423,26 +423,26 @@ public: constexpr time_t TimeT() const noexcept { return (time_t)Seconds(); - } - + } + inline struct timeval TimeVal() const noexcept { - struct timeval tv; - ::Zero(tv); - tv.tv_sec = TimeT(); - tv.tv_usec = MicroSecondsOfSecond(); - return tv; - } - + struct timeval tv; + ::Zero(tv); + tv.tv_sec = TimeT(); + tv.tv_usec = MicroSecondsOfSecond(); + return tv; + } + inline struct tm* LocalTime(struct tm* tm) const noexcept { - time_t clock = Seconds(); - return localtime_r(&clock, tm); - } + time_t clock = Seconds(); + return localtime_r(&clock, tm); + } inline struct tm* GmTime(struct tm* tm) const noexcept { - time_t clock = Seconds(); + time_t clock = Seconds(); return GmTimeR(&clock, tm); - } - + } + /** * Formats the instant using the UTC time zone, with microsecond precision. * @@ -464,7 +464,7 @@ public: * @returns An ISO 8601 formatted string, e.g. '2015-11-21T23:30:27Z'. */ TString ToStringUpToSeconds() const; - + /** * Formats the instant using the system time zone, with microsecond precision. * @@ -555,8 +555,8 @@ public: inline TInstant& operator-=(const T& t) noexcept { return (*this = (*this - t)); } -}; - +}; + Y_DECLARE_PODTYPE(TInstant); template <> @@ -631,22 +631,22 @@ static constexpr bool operator>=(const TTimeBase<S>& l, const TTimeBase<S>& r) n return l.GetValue() >= r.GetValue(); } -namespace NDateTimeHelpers { - template <typename T> +namespace NDateTimeHelpers { + template <typename T> static constexpr T SumWithSaturation(T a, T b) { static_assert(!std::numeric_limits<T>::is_signed, "expect !std::numeric_limits<T>::is_signed"); return Max<T>() - a < b ? Max<T>() : a + b; - } - - template <typename T> + } + + template <typename T> static constexpr T DiffWithSaturation(T a, T b) { static_assert(!std::numeric_limits<T>::is_signed, "expect !std::numeric_limits<T>::is_signed"); return a < b ? 0 : a - b; - } -} - + } +} + constexpr TDuration operator-(const TInstant& l, const TInstant& r) noexcept { return TDuration::FromValue(::NDateTimeHelpers::DiffWithSaturation(l.GetValue(), r.GetValue())); } @@ -804,20 +804,20 @@ constexpr double operator/(const TDuration& x, const TDuration& y) noexcept { } inline TInstant TDuration::ToDeadLine() const { - return ToDeadLine(TInstant::Now()); -} - + return ToDeadLine(TInstant::Now()); +} + constexpr TInstant TDuration::ToDeadLine(TInstant now) const { return now + *this; -} - -void Sleep(TDuration duration); +} + +void Sleep(TDuration duration); void SleepUntil(TInstant instant); - + static inline TInstant Now() noexcept { - return TInstant::Now(); -} - + return TInstant::Now(); +} + #ifdef _MSC_VER #pragma warning(pop) #endif // _MSC_VER diff --git a/util/datetime/base_ut.cpp b/util/datetime/base_ut.cpp index 18a7d41efe..afc3f802eb 100644 --- a/util/datetime/base_ut.cpp +++ b/util/datetime/base_ut.cpp @@ -1,15 +1,15 @@ #include "base.h" #include <library/cpp/testing/unittest/registar.h> - + #include <util/generic/utility.h> #include <util/generic/ylimits.h> #include <util/generic/ymath.h> #include <util/string/cast.h> #include <util/stream/output.h> -#include <util/system/compat.h> +#include <util/system/compat.h> #include <util/random/random.h> - + #include <limits.h> using namespace std::chrono_literals; @@ -220,18 +220,18 @@ Y_UNIT_TEST_SUITE(TDateTimeTest) { UNIT_ASSERT(CompareTM(e.Tm_, t)); /* - * strptime seems to be broken on Mac OS X: - * - * struct tm t; - * char *ret = strptime("Jul", "%b ", &t); - * printf("-%s-\n", ret); - * - * yields "- -": ret contains a pointer to a substring of the format string, - * that should never occur: function returns either NULL or pointer to buf substring. - * - * So this test fails on Mac OS X. - */ - + * strptime seems to be broken on Mac OS X: + * + * struct tm t; + * char *ret = strptime("Jul", "%b ", &t); + * printf("-%s-\n", ret); + * + * yields "- -": ret contains a pointer to a substring of the format string, + * that should never occur: function returns either NULL or pointer to buf substring. + * + * So this test fails on Mac OS X. + */ + struct tm t2; Zero(t2); char* ret = strptime(e.Date_, "%a %b %d %H:%M:%S %Y\n ", &t2); @@ -272,7 +272,7 @@ Y_UNIT_TEST_SUITE(TDateTimeTest) { UNIT_ASSERT(Abs(milliseconds - microseconds / 1000) < 100); UNIT_ASSERT(seconds > 1243008607); // > time when test was written } - + Y_UNIT_TEST(TestStrftime) { struct tm tm; Zero(tm); @@ -281,14 +281,14 @@ Y_UNIT_TEST_SUITE(TDateTimeTest) { tm.tm_mday = 29; UNIT_ASSERT_STRINGS_EQUAL("2009-05-29", Strftime("%Y-%m-%d", &tm)); } - + Y_UNIT_TEST(TestNanoSleep) { NanoSleep(0); NanoSleep(1); NanoSleep(1000); NanoSleep(1000000); } - + static bool TimeZoneEq(const char* zone0, const char* zone1) { if (strcmp(zone0, "GMT") == 0) { zone0 = "UTC"; @@ -340,37 +340,37 @@ Y_UNIT_TEST_SUITE(TDateTimeTest) { Y_UNIT_TEST_SUITE(DateTimeTest) { Y_UNIT_TEST(TestDurationFromFloat) { - UNIT_ASSERT_EQUAL(TDuration::MilliSeconds(500), TDuration::Seconds(0.5)); - UNIT_ASSERT_EQUAL(TDuration::MilliSeconds(500), TDuration::Seconds(0.5f)); - } - + UNIT_ASSERT_EQUAL(TDuration::MilliSeconds(500), TDuration::Seconds(0.5)); + UNIT_ASSERT_EQUAL(TDuration::MilliSeconds(500), TDuration::Seconds(0.5f)); + } + Y_UNIT_TEST(TestSecondsLargeValue) { - unsigned int seconds = UINT_MAX; + unsigned int seconds = UINT_MAX; UNIT_ASSERT_VALUES_EQUAL(((ui64)seconds) * 1000000, TDuration::Seconds(seconds).MicroSeconds()); - } - + } + Y_UNIT_TEST(TestToString) { #define CHECK_CONVERTIBLE(v) \ do { \ UNIT_ASSERT_VALUES_EQUAL(v, ToString(TDuration::Parse(v))); \ UNIT_ASSERT_VALUES_EQUAL(v, TDuration::Parse(v).ToString()); \ } while (0) -#if 0 - - CHECK_CONVERTIBLE("10s"); - CHECK_CONVERTIBLE("1234s"); - CHECK_CONVERTIBLE("1234ms"); - CHECK_CONVERTIBLE("12ms"); - CHECK_CONVERTIBLE("12us"); - CHECK_CONVERTIBLE("1234us"); -#endif - - CHECK_CONVERTIBLE("1.000000s"); - CHECK_CONVERTIBLE("11234.000000s"); - CHECK_CONVERTIBLE("0.011122s"); - CHECK_CONVERTIBLE("33.011122s"); - } - +#if 0 + + CHECK_CONVERTIBLE("10s"); + CHECK_CONVERTIBLE("1234s"); + CHECK_CONVERTIBLE("1234ms"); + CHECK_CONVERTIBLE("12ms"); + CHECK_CONVERTIBLE("12us"); + CHECK_CONVERTIBLE("1234us"); +#endif + + CHECK_CONVERTIBLE("1.000000s"); + CHECK_CONVERTIBLE("11234.000000s"); + CHECK_CONVERTIBLE("0.011122s"); + CHECK_CONVERTIBLE("33.011122s"); + } + Y_UNIT_TEST(TestFromString) { static const struct T { const char* const Str; @@ -404,31 +404,31 @@ Y_UNIT_TEST_SUITE(DateTimeTest) { } Y_UNIT_TEST(TestSleep) { - // check does not throw - Sleep(TDuration::Seconds(0)); - Sleep(TDuration::MicroSeconds(1)); - Sleep(TDuration::MilliSeconds(1)); - } - + // check does not throw + Sleep(TDuration::Seconds(0)); + Sleep(TDuration::MicroSeconds(1)); + Sleep(TDuration::MilliSeconds(1)); + } + Y_UNIT_TEST(TestInstantToString) { UNIT_ASSERT_VALUES_EQUAL(TString("2009-08-06T15:19:06.023455Z"), ToString(TInstant::Seconds(1249571946) + TDuration::MicroSeconds(23455))); UNIT_ASSERT_VALUES_EQUAL(TString("2009-08-06T15:19:06.023455Z"), (TInstant::Seconds(1249571946) + TDuration::MicroSeconds(23455)).ToString()); UNIT_ASSERT_VALUES_EQUAL(TString("2009-08-06T15:19:06Z"), (TInstant::Seconds(1249571946) + TDuration::MicroSeconds(23455)).ToStringUpToSeconds()); - } - + } + Y_UNIT_TEST(TestInstantToRfc822String) { UNIT_ASSERT_VALUES_EQUAL(TString("Thu, 06 Aug 2009 15:19:06 GMT"), (TInstant::Seconds(1249571946) + TDuration::MicroSeconds(23455)).ToRfc822String()); } Y_UNIT_TEST(TestInstantMath) { - UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(1719), TInstant::Seconds(1700) + TDuration::Seconds(19)); - // overflow - UNIT_ASSERT_VALUES_EQUAL(TInstant::Max(), TInstant::Max() - TDuration::Seconds(10) + TDuration::Seconds(19)); - // underflow - UNIT_ASSERT_VALUES_EQUAL(TInstant::Zero(), TInstant::Seconds(1000) - TDuration::Seconds(2000)); - UNIT_ASSERT_VALUES_EQUAL(TDuration::Zero(), TInstant::Seconds(1000) - TInstant::Seconds(2000)); - } - + UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(1719), TInstant::Seconds(1700) + TDuration::Seconds(19)); + // overflow + UNIT_ASSERT_VALUES_EQUAL(TInstant::Max(), TInstant::Max() - TDuration::Seconds(10) + TDuration::Seconds(19)); + // underflow + UNIT_ASSERT_VALUES_EQUAL(TInstant::Zero(), TInstant::Seconds(1000) - TDuration::Seconds(2000)); + UNIT_ASSERT_VALUES_EQUAL(TDuration::Zero(), TInstant::Seconds(1000) - TInstant::Seconds(2000)); + } + Y_UNIT_TEST(TestDurationMath) { TDuration empty; UNIT_ASSERT(!empty); @@ -439,14 +439,14 @@ Y_UNIT_TEST_SUITE(DateTimeTest) { TDuration nonEmpty = TDuration::MicroSeconds(1); UNIT_ASSERT(nonEmpty); - UNIT_ASSERT_VALUES_EQUAL(TDuration::Seconds(110), TDuration::Seconds(77) + TDuration::Seconds(33)); - // overflow - UNIT_ASSERT_VALUES_EQUAL(TDuration::Max(), TDuration::Max() - TDuration::Seconds(1) + TDuration::Seconds(10)); - // underflow - UNIT_ASSERT_VALUES_EQUAL(TDuration::Zero(), TDuration::Seconds(20) - TDuration::Seconds(200)); + UNIT_ASSERT_VALUES_EQUAL(TDuration::Seconds(110), TDuration::Seconds(77) + TDuration::Seconds(33)); + // overflow + UNIT_ASSERT_VALUES_EQUAL(TDuration::Max(), TDuration::Max() - TDuration::Seconds(1) + TDuration::Seconds(10)); + // underflow + UNIT_ASSERT_VALUES_EQUAL(TDuration::Zero(), TDuration::Seconds(20) - TDuration::Seconds(200)); // division UNIT_ASSERT_DOUBLES_EQUAL(TDuration::Minutes(1) / TDuration::Seconds(10), 6.0, 1e-9); - } + } Y_UNIT_TEST(TestDurationGetters) { const TDuration value = TDuration::MicroSeconds(1234567); @@ -651,4 +651,4 @@ Y_UNIT_TEST_SUITE(DateTimeTest) { static_assert(TDuration::Zero() + 1s == 1s); static_assert(TInstant::Seconds(1) + 1s == TInstant::Seconds(2)); } -} +} diff --git a/util/datetime/cputimer.cpp b/util/datetime/cputimer.cpp index 0d60296836..516d372c37 100644 --- a/util/datetime/cputimer.cpp +++ b/util/datetime/cputimer.cpp @@ -30,32 +30,32 @@ TTimer::~TTimer() { Cerr << Message_.Str(); } -static ui64 ManuallySetCyclesPerSecond = 0; - -static ui64 GetCyclesPerSecond() { +static ui64 ManuallySetCyclesPerSecond = 0; + +static ui64 GetCyclesPerSecond() { if (ManuallySetCyclesPerSecond != 0) { - return ManuallySetCyclesPerSecond; + return ManuallySetCyclesPerSecond; } else { return NHPTimer::GetCyclesPerSecond(); } } -void SetCyclesPerSecond(ui64 cycles) { - ManuallySetCyclesPerSecond = cycles; -} - -ui64 GetCyclesPerMillisecond() { - return GetCyclesPerSecond() / 1000; -} - -TDuration CyclesToDuration(ui64 cycles) { - return TDuration::MicroSeconds(cycles * 1000000 / GetCyclesPerSecond()); -} - -ui64 DurationToCycles(TDuration duration) { - return duration.MicroSeconds() * GetCyclesPerSecond() / 1000000; -} - +void SetCyclesPerSecond(ui64 cycles) { + ManuallySetCyclesPerSecond = cycles; +} + +ui64 GetCyclesPerMillisecond() { + return GetCyclesPerSecond() / 1000; +} + +TDuration CyclesToDuration(ui64 cycles) { + return TDuration::MicroSeconds(cycles * 1000000 / GetCyclesPerSecond()); +} + +ui64 DurationToCycles(TDuration duration) { + return duration.MicroSeconds() * GetCyclesPerSecond() / 1000000; +} + TPrecisionTimer::TPrecisionTimer() : Start(::GetCycleCount()) { diff --git a/util/datetime/cputimer.h b/util/datetime/cputimer.h index 1704b55dcc..7d38d5bdb3 100644 --- a/util/datetime/cputimer.h +++ b/util/datetime/cputimer.h @@ -2,13 +2,13 @@ #include "base.h" -#include <util/system/rusage.h> +#include <util/system/rusage.h> #include <util/generic/string.h> #include <util/stream/str.h> class TTimer { private: - TInstant Start_; + TInstant Start_; TStringStream Message_; public: @@ -17,51 +17,51 @@ public: }; class TSimpleTimer { - TInstant T; + TInstant T; public: TSimpleTimer() { Reset(); } TDuration Get() const { - return TInstant::Now() - T; + return TInstant::Now() - T; } void Reset() { - T = TInstant::Now(); + T = TInstant::Now(); } }; class TProfileTimer { - TDuration T; + TDuration T; public: TProfileTimer() { Reset(); } TDuration Get() const { - return TRusage::Get().Utime - T; + return TRusage::Get().Utime - T; } TDuration Step() { - TRusage r; - r.Fill(); - TDuration d = r.Utime - T; - T = r.Utime; + TRusage r; + r.Fill(); + TDuration d = r.Utime - T; + T = r.Utime; return d; } void Reset() { - T = TRusage::Get().Utime; + T = TRusage::Get().Utime; } }; -/// Return cached processor cycle count per second. Method takes 1 second at first invocation. -/// Note, on older systems cycle rate may change during program lifetime, -/// so returned value may be incorrect. Modern Intel and AMD processors keep constant TSC rate. +/// Return cached processor cycle count per second. Method takes 1 second at first invocation. +/// Note, on older systems cycle rate may change during program lifetime, +/// so returned value may be incorrect. Modern Intel and AMD processors keep constant TSC rate. ui64 GetCyclesPerMillisecond(); -void SetCyclesPerSecond(ui64 cycles); +void SetCyclesPerSecond(ui64 cycles); + +TDuration CyclesToDuration(ui64 cycles); +ui64 DurationToCycles(TDuration duration); -TDuration CyclesToDuration(ui64 cycles); -ui64 DurationToCycles(TDuration duration); - class TPrecisionTimer { private: ui64 Start = 0; @@ -91,7 +91,7 @@ public: ~TFuncTimer(); private: - const TInstant Start_; + const TInstant Start_; const char* Func_; }; diff --git a/util/datetime/parser.h b/util/datetime/parser.h index 7816bfd349..f0c1b4a0c7 100644 --- a/util/datetime/parser.h +++ b/util/datetime/parser.h @@ -1,160 +1,160 @@ #pragma once - -// probably you do not need to include this file directly, use "util/datetime/base.h" - -#include "base.h" - -struct TDateTimeFields { - TDateTimeFields() { - Zero(*this); + +// probably you do not need to include this file directly, use "util/datetime/base.h" + +#include "base.h" + +struct TDateTimeFields { + TDateTimeFields() { + Zero(*this); ZoneOffsetMinutes = 0; Hour = 0; - } - - ui32 Year; - ui32 Month; // 1..12 - ui32 Day; // 1 .. 31 - ui32 Hour; // 0 .. 23 - ui32 Minute; // 0 .. 59 - ui32 Second; // 0 .. 60 - ui32 MicroSecond; // 0 .. 999999 - i32 ZoneOffsetMinutes; - - void SetLooseYear(ui32 year) { - if (year < 60) - year += 100; - if (year < 160) - year += 1900; - Year = year; - } - + } + + ui32 Year; + ui32 Month; // 1..12 + ui32 Day; // 1 .. 31 + ui32 Hour; // 0 .. 23 + ui32 Minute; // 0 .. 59 + ui32 Second; // 0 .. 60 + ui32 MicroSecond; // 0 .. 999999 + i32 ZoneOffsetMinutes; + + void SetLooseYear(ui32 year) { + if (year < 60) + year += 100; + if (year < 160) + year += 1900; + Year = year; + } + bool IsOk() const noexcept { - if (Year < 1970) - return false; - if (Month < 1 || Month > 12) - return false; - - unsigned int maxMonthDay = 31; - if (Month == 4 || Month == 6 || Month == 9 || Month == 11) { - maxMonthDay = 30; - } else if (Month == 2) { - if (Year % 4 == 0 && (Year % 100 != 0 || Year % 400 == 0)) - // leap year - maxMonthDay = 29; - else - maxMonthDay = 28; - } - if (Day > maxMonthDay) - return false; - - if (Hour > 23) - return false; - - if (Minute > 59) - return false; - + if (Year < 1970) + return false; + if (Month < 1 || Month > 12) + return false; + + unsigned int maxMonthDay = 31; + if (Month == 4 || Month == 6 || Month == 9 || Month == 11) { + maxMonthDay = 30; + } else if (Month == 2) { + if (Year % 4 == 0 && (Year % 100 != 0 || Year % 400 == 0)) + // leap year + maxMonthDay = 29; + else + maxMonthDay = 28; + } + if (Day > maxMonthDay) + return false; + + if (Hour > 23) + return false; + + if (Minute > 59) + return false; + // handle leap second which is explicitly allowed by ISO 8601:2004(E) $2.2.2 // https://datatracker.ietf.org/doc/html/rfc3339#section-5.6 - if (Second > 60) - return false; - - if (MicroSecond > 999999) - return false; - + if (Second > 60) + return false; + + if (MicroSecond > 999999) + return false; + if (Year == 1970 && Month == 1 && Day == 1) { if ((i64)(3600 * Hour + 60 * Minute + Second) < (60 * ZoneOffsetMinutes)) return false; } - return true; - } - + return true; + } + TInstant ToInstant(TInstant defaultValue) const { - time_t tt = ToTimeT(-1); - if (tt == -1) - return defaultValue; - return TInstant::Seconds(tt) + TDuration::MicroSeconds(MicroSecond); - } - + time_t tt = ToTimeT(-1); + if (tt == -1) + return defaultValue; + return TInstant::Seconds(tt) + TDuration::MicroSeconds(MicroSecond); + } + time_t ToTimeT(time_t defaultValue) const { - if (!IsOk()) - return defaultValue; - struct tm tm; - Zero(tm); - tm.tm_year = Year - 1900; - tm.tm_mon = Month - 1; - tm.tm_mday = Day; - tm.tm_hour = Hour; - tm.tm_min = Minute; - tm.tm_sec = Second; + if (!IsOk()) + return defaultValue; + struct tm tm; + Zero(tm); + tm.tm_year = Year - 1900; + tm.tm_mon = Month - 1; + tm.tm_mday = Day; + tm.tm_hour = Hour; + tm.tm_min = Minute; + tm.tm_sec = Second; time_t tt = TimeGM(&tm); - if (tt == -1) - return defaultValue; - return tt - ZoneOffsetMinutes * 60; - } -}; - -class TDateTimeParserBase { + if (tt == -1) + return defaultValue; + return tt - ZoneOffsetMinutes * 60; + } +}; + +class TDateTimeParserBase { public: const TDateTimeFields& GetDateTimeFields() const { return DateTimeFields; } -protected: - TDateTimeFields DateTimeFields; +protected: + TDateTimeFields DateTimeFields; int cs; //for ragel - int Sign; - int I; - int Dc; - + int Sign; + int I; + int Dc; + protected: - TDateTimeParserBase() + TDateTimeParserBase() : DateTimeFields() , cs(0) , Sign(0) , I(0xDEADBEEF) // to guarantee unittest break if ragel code is incorrect , Dc(0xDEADBEEF) - { - } + { + } inline TInstant GetResult(int firstFinalState, TInstant defaultValue) const { if (cs < firstFinalState) return defaultValue; return DateTimeFields.ToInstant(defaultValue); } -}; - +}; + #define DECLARE_PARSER(CLASS) \ struct CLASS: public TDateTimeParserBase { \ CLASS(); \ bool ParsePart(const char* input, size_t len); \ TInstant GetResult(TInstant defaultValue) const; \ - }; - -DECLARE_PARSER(TIso8601DateTimeParser) -DECLARE_PARSER(TRfc822DateTimeParser) -DECLARE_PARSER(THttpDateTimeParser) + }; + +DECLARE_PARSER(TIso8601DateTimeParser) +DECLARE_PARSER(TRfc822DateTimeParser) +DECLARE_PARSER(THttpDateTimeParser) DECLARE_PARSER(TX509ValidityDateTimeParser) DECLARE_PARSER(TX509Validity4yDateTimeParser) - -#undef DECLARE_PARSER - -struct TDurationParser { - int cs; - - ui64 I; - ui32 Dc; - - i32 MultiplierPower; // 6 for seconds, 0 for microseconds, -3 for nanoseconds + +#undef DECLARE_PARSER + +struct TDurationParser { + int cs; + + ui64 I; + ui32 Dc; + + i32 MultiplierPower; // 6 for seconds, 0 for microseconds, -3 for nanoseconds i32 Multiplier; - ui64 IntegerPart; - ui32 FractionPart; - ui32 FractionDigits; - - TDurationParser(); - bool ParsePart(const char* input, size_t len); - TDuration GetResult(TDuration defaultValue) const; -}; + ui64 IntegerPart; + ui32 FractionPart; + ui32 FractionDigits; + + TDurationParser(); + bool ParsePart(const char* input, size_t len); + TDuration GetResult(TDuration defaultValue) const; +}; /** Depcrecated cause of default hour offset (+4 hours) diff --git a/util/datetime/parser.rl6 b/util/datetime/parser.rl6 index 5a8e3f48ad..931f09eae1 100644 --- a/util/datetime/parser.rl6 +++ b/util/datetime/parser.rl6 @@ -11,23 +11,23 @@ %%{ -machine DateTimeParserCommon; +machine DateTimeParserCommon; + +sp = ' '; -sp = ' '; - action clear_int { - I = 0; - Dc = 0; + I = 0; + Dc = 0; } action update_int { - I = I * 10 + (fc - '0'); - ++Dc; + I = I * 10 + (fc - '0'); + ++Dc; } -int = (digit+) - >clear_int - $update_int; +int = (digit+) + >clear_int + $update_int; int1 = digit >clear_int @@ -37,10 +37,10 @@ int2 = (digit digit) >clear_int $update_int; -int3 = (digit digit digit) - >clear_int - $update_int; - +int3 = (digit digit digit) + >clear_int + $update_int; + int4 = (digit digit digit digit) >clear_int $update_int; @@ -53,60 +53,60 @@ int24 = ( digit digit ( digit digit )? ) >clear_int $update_int; -# According to both RFC2822 and RFC2616 dates MUST be case-sensitive, -# but Andrey fomichev@ wants relaxed parser - -month3 = - 'Jan'i %{ DateTimeFields.Month = 1; } - | 'Feb'i %{ DateTimeFields.Month = 2; } - | 'Mar'i %{ DateTimeFields.Month = 3; } - | 'Apr'i %{ DateTimeFields.Month = 4; } - | 'May'i %{ DateTimeFields.Month = 5; } - | 'Jun'i %{ DateTimeFields.Month = 6; } - | 'Jul'i %{ DateTimeFields.Month = 7; } - | 'Aug'i %{ DateTimeFields.Month = 8; } - | 'Sep'i %{ DateTimeFields.Month = 9; } - | 'Oct'i %{ DateTimeFields.Month = 10; } - | 'Nov'i %{ DateTimeFields.Month = 11; } - | 'Dec'i %{ DateTimeFields.Month = 12; }; - -wkday = 'Mon'i | 'Tue'i | 'Wed'i | 'Thu'i | 'Fri'i | 'Sat'i | 'Sun'i; -weekday = 'Monday'i | 'Tuesday'i | 'Wednesday'i | 'Thursday'i - | 'Friday'i | 'Saturday'i | 'Sunday'i; - -action set_second { DateTimeFields.Second = I; } -action set_minute { DateTimeFields.Minute = I; } -action set_hour { DateTimeFields.Hour = I; } -action set_day { DateTimeFields.Day = I; } -action set_month { DateTimeFields.Month = I; } -action set_year { DateTimeFields.SetLooseYear(I); } -action set_zone_utc { DateTimeFields.ZoneOffsetMinutes = 0; } - -}%% - -%%{ - -machine RFC822DateParser; - -################# RFC 2822 3.3 Full Date ################### - -include DateTimeParserCommon; - -ws1 = (space+); -ws0 = (space*); -dow_spec = ( wkday ',' )?; - -day = int12 %set_day; +# According to both RFC2822 and RFC2616 dates MUST be case-sensitive, +# but Andrey fomichev@ wants relaxed parser + +month3 = + 'Jan'i %{ DateTimeFields.Month = 1; } + | 'Feb'i %{ DateTimeFields.Month = 2; } + | 'Mar'i %{ DateTimeFields.Month = 3; } + | 'Apr'i %{ DateTimeFields.Month = 4; } + | 'May'i %{ DateTimeFields.Month = 5; } + | 'Jun'i %{ DateTimeFields.Month = 6; } + | 'Jul'i %{ DateTimeFields.Month = 7; } + | 'Aug'i %{ DateTimeFields.Month = 8; } + | 'Sep'i %{ DateTimeFields.Month = 9; } + | 'Oct'i %{ DateTimeFields.Month = 10; } + | 'Nov'i %{ DateTimeFields.Month = 11; } + | 'Dec'i %{ DateTimeFields.Month = 12; }; + +wkday = 'Mon'i | 'Tue'i | 'Wed'i | 'Thu'i | 'Fri'i | 'Sat'i | 'Sun'i; +weekday = 'Monday'i | 'Tuesday'i | 'Wednesday'i | 'Thursday'i + | 'Friday'i | 'Saturday'i | 'Sunday'i; + +action set_second { DateTimeFields.Second = I; } +action set_minute { DateTimeFields.Minute = I; } +action set_hour { DateTimeFields.Hour = I; } +action set_day { DateTimeFields.Day = I; } +action set_month { DateTimeFields.Month = I; } +action set_year { DateTimeFields.SetLooseYear(I); } +action set_zone_utc { DateTimeFields.ZoneOffsetMinutes = 0; } + +}%% + +%%{ + +machine RFC822DateParser; + +################# RFC 2822 3.3 Full Date ################### + +include DateTimeParserCommon; + +ws1 = (space+); +ws0 = (space*); +dow_spec = ( wkday ',' )?; + +day = int12 %set_day; year = int24 %set_year; # actually it must be from 0 to 23 -hour = int2 %set_hour; +hour = int2 %set_hour; # actually it must be from 0 to 59 -min = int2 %set_minute; +min = int2 %set_minute; # actually it must be from 0 to 59 -sec = int2 %set_second; +sec = int2 %set_second; sec_spec = ( ':' . sec )?; @@ -114,7 +114,7 @@ sec_spec = ( ':' . sec )?; action set_mil_offset { char c = (char)toupper(fc); if (c == 'Z') - DateTimeFields.ZoneOffsetMinutes = 0; + DateTimeFields.ZoneOffsetMinutes = 0; else { if (c <= 'M') { // ['A'..'M'] \ 'J' @@ -140,8 +140,8 @@ mil_zone = /[A-IK-Za-ik-z]/ $set_mil_offset; # it is a bug in ragel 5 because ragel 6.2 works correctly with % at the end of string. # see http://www.complang.org/ragel/ChangeLog. -zone = 'UT' @{ DateTimeFields.ZoneOffsetMinutes = 0; } - | 'GMT' @{ DateTimeFields.ZoneOffsetMinutes = 0; } +zone = 'UT' @{ DateTimeFields.ZoneOffsetMinutes = 0; } + | 'GMT' @{ DateTimeFields.ZoneOffsetMinutes = 0; } | 'EST' @{ DateTimeFields.ZoneOffsetMinutes = -(i32)TDuration::Hours(5).Minutes();} | 'EDT' @{ DateTimeFields.ZoneOffsetMinutes = -(i32)TDuration::Hours(4).Minutes(); } | 'CST' @{ DateTimeFields.ZoneOffsetMinutes = -(i32)TDuration::Hours(6).Minutes();} @@ -151,29 +151,29 @@ zone = 'UT' @{ DateTimeFields.ZoneOffsetMinutes = 0; } | 'PST' @{ DateTimeFields.ZoneOffsetMinutes = -(i32)TDuration::Hours(8).Minutes();} | 'PDT' @{ DateTimeFields.ZoneOffsetMinutes = -(i32)TDuration::Hours(7).Minutes(); }; -digit_offset = ('+' | '-') > { Sign = fc == '+' ? 1 : -1; } . int4 @set_digit_offset; +digit_offset = ('+' | '-') > { Sign = fc == '+' ? 1 : -1; } . int4 @set_digit_offset; offset = ( zone | mil_zone | digit_offset ); -rfc822datetime = ws0 . dow_spec . ws0 . day . ws1 . month3 . ws1 . year . ws1 . hour . ':' . min . sec_spec . ws1 . offset . ws0; +rfc822datetime = ws0 . dow_spec . ws0 . day . ws1 . month3 . ws1 . year . ws1 . hour . ':' . min . sec_spec . ws1 . offset . ws0; -main := rfc822datetime; +main := rfc822datetime; write data noerror; }%% TRfc822DateTimeParserDeprecated::TRfc822DateTimeParserDeprecated() { - %% write init; -} + %% write init; +} bool TRfc822DateTimeParserDeprecated::ParsePart(const char* input, size_t len) { - const char* p = input; - const char* pe = input + len; + const char* p = input; + const char* pe = input + len; - %% write exec; - return cs != %%{ write error; }%%; -} + %% write exec; + return cs != %%{ write error; }%%; +} TRfc822DateTimeParser::TRfc822DateTimeParser() { %% write init; @@ -187,53 +187,53 @@ bool TRfc822DateTimeParser::ParsePart(const char* input, size_t len) { return cs != %%{ write error; }%%; } -%%{ - -machine ISO8601DateTimeParser; - -include DateTimeParserCommon; - -year = int4 @set_year; -month = int2 @set_month; -day = int2 @set_day; -hour = int2 @set_hour; -minute = int2 @set_minute; -second = int2 @set_second; +%%{ + +machine ISO8601DateTimeParser; + +include DateTimeParserCommon; + +year = int4 @set_year; +month = int2 @set_month; +day = int2 @set_day; +hour = int2 @set_hour; +minute = int2 @set_minute; +second = int2 @set_second; secondFrac = digit {1,6} >clear_int $update_int @{ - ui32 us = I; - for (int k = Dc; k < 6; ++k) { - us *= 10; - } - DateTimeFields.MicroSecond = us; -}; + ui32 us = I; + for (int k = Dc; k < 6; ++k) { + us *= 10; + } + DateTimeFields.MicroSecond = us; +}; secondFracTail = (digit*); - -zoneZ = [Zz] @set_zone_utc; + +zoneZ = [Zz] @set_zone_utc; zoneOffset = space? . ('+' | '-') >{ Sign = fc == '+' ? 1 : -1; } . int2 @{ DateTimeFields.ZoneOffsetMinutes = Sign * (i32)TDuration::Hours(I).Minutes(); } . (':')? . (int2 @{ DateTimeFields.ZoneOffsetMinutes += I * Sign; })?; -zone = zoneZ | zoneOffset; - -iso8601date = (year . '-' . month . '-' . day) | (year . month . day); +zone = zoneZ | zoneOffset; + +iso8601date = (year . '-' . month . '-' . day) | (year . month . day); iso8601time = (hour . ':' . minute . (':' . second ('.' secondFrac secondFracTail)?)?) | (hour . minute . second?); - -iso8601datetime = iso8601date . ([Tt ] . iso8601time . zone?)?; - -main := iso8601datetime; - -write data noerror; - -}%% - + +iso8601datetime = iso8601date . ([Tt ] . iso8601time . zone?)?; + +main := iso8601datetime; + +write data noerror; + +}%% + TIso8601DateTimeParserDeprecated::TIso8601DateTimeParserDeprecated() { %% write init; -} - +} + bool TIso8601DateTimeParserDeprecated::ParsePart(const char* input, size_t len) { - const char* p = input; - const char* pe = input + len; - + const char* p = input; + const char* pe = input + len; + %% write exec; - return cs != %%{ write error; }%%; -} + return cs != %%{ write error; }%%; +} TIso8601DateTimeParser::TIso8601DateTimeParser() { %% write init; @@ -247,50 +247,50 @@ bool TIso8601DateTimeParser::ParsePart(const char* input, size_t len) { return cs != %%{ write error; }%%; } -%%{ - -machine HttpDateTimeParser; - -include DateTimeParserCommon; - -################# RFC 2616 3.3.1 Full Date ################# - -time = int2 %set_hour ':' int2 %set_minute ':' int2 %set_second; -date1 = int2 %set_day ' ' month3 ' ' int4 %set_year; -date2 = int2 %set_day '-' month3 '-' int2 %set_year; -date3 = month3 sp (int2 | sp int1) %set_day; - -rfc1123_date = wkday ',' sp date1 sp time sp 'GMT'i; -rfc850_date = weekday ',' sp date2 sp time sp 'GMT'i; -asctime_date = wkday sp date3 sp time sp int4 @set_year; -http_date = (rfc1123_date | rfc850_date | asctime_date) @set_zone_utc; - -}%% - -%%{ - -machine HttpDateTimeParserStandalone; - -include HttpDateTimeParser; - -main := http_date; - -write data noerror; - -}%% - +%%{ + +machine HttpDateTimeParser; + +include DateTimeParserCommon; + +################# RFC 2616 3.3.1 Full Date ################# + +time = int2 %set_hour ':' int2 %set_minute ':' int2 %set_second; +date1 = int2 %set_day ' ' month3 ' ' int4 %set_year; +date2 = int2 %set_day '-' month3 '-' int2 %set_year; +date3 = month3 sp (int2 | sp int1) %set_day; + +rfc1123_date = wkday ',' sp date1 sp time sp 'GMT'i; +rfc850_date = weekday ',' sp date2 sp time sp 'GMT'i; +asctime_date = wkday sp date3 sp time sp int4 @set_year; +http_date = (rfc1123_date | rfc850_date | asctime_date) @set_zone_utc; + +}%% + +%%{ + +machine HttpDateTimeParserStandalone; + +include HttpDateTimeParser; + +main := http_date; + +write data noerror; + +}%% + THttpDateTimeParserDeprecated::THttpDateTimeParserDeprecated() { - %% write init; -} - + %% write init; +} + bool THttpDateTimeParserDeprecated::ParsePart(const char* input, size_t len) { - const char* p = input; - const char* pe = input + len; - - %% write exec; - return cs != %%{ write error; }%%; -} - + const char* p = input; + const char* pe = input + len; + + %% write exec; + return cs != %%{ write error; }%%; +} + THttpDateTimeParser::THttpDateTimeParser() { %% write init; } @@ -334,7 +334,7 @@ bool TX509ValidityDateTimeParserDeprecated::ParsePart(const char *input, size_t const char *pe = input + len; %% write exec; - return cs != %%{ write error; }%%; + return cs != %%{ write error; }%%; } TX509ValidityDateTimeParser::TX509ValidityDateTimeParser() { @@ -379,7 +379,7 @@ bool TX509Validity4yDateTimeParserDeprecated::ParsePart(const char *input, size_ const char *pe = input + len; %% write exec; - return cs != %%{ write error; }%%; + return cs != %%{ write error; }%%; } TX509Validity4yDateTimeParser::TX509Validity4yDateTimeParser() { @@ -397,23 +397,23 @@ bool TX509Validity4yDateTimeParser::ParsePart(const char *input, size_t len) { TInstant TIso8601DateTimeParserDeprecated::GetResult(TInstant defaultValue) const { Y_UNUSED(ISO8601DateTimeParser_en_main); return TDateTimeParserBaseDeprecated::GetResult(ISO8601DateTimeParser_first_final, defaultValue); -} - +} + TInstant TRfc822DateTimeParserDeprecated::GetResult(TInstant defaultValue) const { Y_UNUSED(RFC822DateParser_en_main); return TDateTimeParserBaseDeprecated::GetResult(RFC822DateParser_first_final, defaultValue); -} - +} + TInstant THttpDateTimeParserDeprecated::GetResult(TInstant defaultValue) const { Y_UNUSED(HttpDateTimeParserStandalone_en_main); return TDateTimeParserBaseDeprecated::GetResult(HttpDateTimeParserStandalone_first_final, defaultValue); -} - +} + TInstant TX509ValidityDateTimeParserDeprecated::GetResult(TInstant defaultValue) const { Y_UNUSED(X509ValidityDateTimeParser_en_main); return TDateTimeParserBaseDeprecated::GetResult(X509ValidityDateTimeParser_first_final, defaultValue); } - + TInstant TX509Validity4yDateTimeParserDeprecated::GetResult(TInstant defaultValue) const { Y_UNUSED(X509Validity4yDateTimeParser_en_main); return TDateTimeParserBaseDeprecated::GetResult(X509Validity4yDateTimeParser_first_final, defaultValue); @@ -444,34 +444,34 @@ TInstant TX509Validity4yDateTimeParser::GetResult(TInstant defaultValue) const { return TDateTimeParserBase::GetResult(X509Validity4yDateTimeParser_first_final, defaultValue); } -template<class TParser, class TResult> -static inline TResult Parse(const char* input, size_t len, TResult defaultValue) { - TParser parser; - if (!parser.ParsePart(input, len)) - return defaultValue; - return parser.GetResult(defaultValue); -} - +template<class TParser, class TResult> +static inline TResult Parse(const char* input, size_t len, TResult defaultValue) { + TParser parser; + if (!parser.ParsePart(input, len)) + return defaultValue; + return parser.GetResult(defaultValue); +} + template<class TParser, class TResult, bool ThrowExceptionOnFailure = true> -static inline TResult ParseUnsafe(const char* input, size_t len) { - TResult r = Parse<TParser, TResult>(input, len, TResult::Max()); +static inline TResult ParseUnsafe(const char* input, size_t len) { + TResult r = Parse<TParser, TResult>(input, len, TResult::Max()); if (ThrowExceptionOnFailure && r == TResult::Max()) ythrow TDateTimeParseException() << "error in datetime parsing. Input data: " << TStringBuf(input, len); - return r; -} - + return r; +} + TInstant TInstant::ParseIso8601Deprecated(const TStringBuf input) { return ParseUnsafe<TIso8601DateTimeParserDeprecated, TInstant>(input.data(), input.size()); -} - +} + TInstant TInstant::ParseRfc822Deprecated(const TStringBuf input) { return ParseUnsafe<TRfc822DateTimeParserDeprecated, TInstant>(input.data(), input.size()); -} - +} + TInstant TInstant::ParseHttpDeprecated(const TStringBuf input) { return ParseUnsafe<THttpDateTimeParserDeprecated, TInstant>(input.data(), input.size()); -} - +} + TInstant TInstant::ParseX509ValidityDeprecated(const TStringBuf input) { switch (input.size()) { case 13: @@ -482,7 +482,7 @@ TInstant TInstant::ParseX509ValidityDeprecated(const TStringBuf input) { ythrow TDateTimeParseException(); } } - + bool TInstant::TryParseIso8601Deprecated(const TStringBuf input, TInstant& instant) { const auto parsed = ParseUnsafe<TIso8601DateTimeParserDeprecated, TInstant, false>(input.data(), input.size()); if (TInstant::Max() == parsed) { @@ -663,44 +663,44 @@ bool ParseRFC822DateTime(const char* input, time_t& utcTime) { return ParseRFC822DateTime(input, strlen(input), utcTime); } -bool ParseISO8601DateTime(const char* input, time_t& utcTime) { - return ParseISO8601DateTime(input, strlen(input), utcTime); -} - -bool ParseHTTPDateTime(const char* input, time_t& utcTime) { - return ParseHTTPDateTime(input, strlen(input), utcTime); -} - +bool ParseISO8601DateTime(const char* input, time_t& utcTime) { + return ParseISO8601DateTime(input, strlen(input), utcTime); +} + +bool ParseHTTPDateTime(const char* input, time_t& utcTime) { + return ParseHTTPDateTime(input, strlen(input), utcTime); +} + bool ParseX509ValidityDateTime(const char* input, time_t& utcTime) { return ParseX509ValidityDateTime(input, strlen(input), utcTime); } - + bool ParseRFC822DateTime(const char* input, size_t inputLen, time_t& utcTime) { - try { + try { utcTime = ParseUnsafe<TRfc822DateTimeParser, TInstant>(input, inputLen).TimeT(); - return true; - } catch (const TDateTimeParseException&) { + return true; + } catch (const TDateTimeParseException&) { return false; - } -} - + } +} + bool ParseISO8601DateTime(const char* input, size_t inputLen, time_t& utcTime) { - try { + try { utcTime = ParseUnsafe<TIso8601DateTimeParser, TInstant>(input, inputLen).TimeT(); return true; - } catch (const TDateTimeParseException&) { - return false; + } catch (const TDateTimeParseException&) { + return false; } } - -bool ParseHTTPDateTime(const char* input, size_t inputLen, time_t& utcTime) { - try { + +bool ParseHTTPDateTime(const char* input, size_t inputLen, time_t& utcTime) { + try { utcTime = ParseUnsafe<THttpDateTimeParser, TInstant>(input, inputLen).TimeT(); - return true; - } catch (const TDateTimeParseException&) { - return false; - } -} + return true; + } catch (const TDateTimeParseException&) { + return false; + } +} bool ParseX509ValidityDateTime(const char* input, size_t inputLen, time_t& utcTime) { TInstant r; @@ -720,15 +720,15 @@ bool ParseX509ValidityDateTime(const char* input, size_t inputLen, time_t& utcTi return true; } -%%{ - -machine TDurationParser; - -include DateTimeParserCommon; - - -multiplier - = '' # >{ MultiplierPower = 6; } # work around Ragel bugs +%%{ + +machine TDurationParser; + +include DateTimeParserCommon; + + +multiplier + = '' # >{ MultiplierPower = 6; } # work around Ragel bugs | 'w' @{ MultiplierPower = 6; Multiplier = 604800; } | 'd' @{ MultiplierPower = 6; Multiplier = 86400; } | 'h' @{ MultiplierPower = 6; Multiplier = 3600; } @@ -737,57 +737,57 @@ multiplier | 'ms' @{ MultiplierPower = 3; Multiplier = 1; } | 'us' @{ MultiplierPower = 0; Multiplier = 1; } | 'ns' @{ MultiplierPower = -3; Multiplier = 1; } - ; - -integer = int @{ IntegerPart = I; }; - -fraction = '.' digit {1,6} >clear_int $update_int @{ FractionPart = I; FractionDigits = Dc; } digit*; - -duration = integer fraction? multiplier; - -main := duration; - -write data noerror; - -}%% - -TDurationParser::TDurationParser() - : cs(0) - , I(0) - , Dc(0) - , MultiplierPower(6) + ; + +integer = int @{ IntegerPart = I; }; + +fraction = '.' digit {1,6} >clear_int $update_int @{ FractionPart = I; FractionDigits = Dc; } digit*; + +duration = integer fraction? multiplier; + +main := duration; + +write data noerror; + +}%% + +TDurationParser::TDurationParser() + : cs(0) + , I(0) + , Dc(0) + , MultiplierPower(6) , Multiplier(1) - , IntegerPart(0) - , FractionPart(0) - , FractionDigits(0) -{ + , IntegerPart(0) + , FractionPart(0) + , FractionDigits(0) +{ Y_UNUSED(TDurationParser_en_main); - %% write init; -} - -bool TDurationParser::ParsePart(const char* input, size_t len) { - const char* p = input; - const char* pe = input + len; - - %% write exec; - return cs != %%{ write error; }%%; -} - + %% write init; +} + +bool TDurationParser::ParsePart(const char* input, size_t len) { + const char* p = input; + const char* pe = input + len; + + %% write exec; + return cs != %%{ write error; }%%; +} + static inline ui64 DecPower(ui64 part, i32 power) { if (power >= 0) return part * Power(10, power); return part / Power(10, -power); -} - -TDuration TDurationParser::GetResult(TDuration defaultValue) const { - if (cs < TDurationParser_first_final) - return defaultValue; - ui64 us = 0; +} + +TDuration TDurationParser::GetResult(TDuration defaultValue) const { + if (cs < TDurationParser_first_final) + return defaultValue; + ui64 us = 0; us += Multiplier * DecPower(IntegerPart, MultiplierPower); us += Multiplier * DecPower(FractionPart, MultiplierPower - FractionDigits); - return TDuration::MicroSeconds(us); -} - + return TDuration::MicroSeconds(us); +} + bool TDuration::TryParse(const TStringBuf input, TDuration& result) { TDuration r = ::Parse<TDurationParser, TDuration>(input.data(), input.size(), TDuration::Max()); if (r == TDuration::Max()) @@ -795,7 +795,7 @@ bool TDuration::TryParse(const TStringBuf input, TDuration& result) { result = r; return true; } - + TDuration TDuration::Parse(const TStringBuf input) { return ParseUnsafe<TDurationParser, TDuration>(input.data(), input.size()); -} +} diff --git a/util/datetime/parser_ut.cpp b/util/datetime/parser_ut.cpp index 0fc4d4993a..61364af997 100644 --- a/util/datetime/parser_ut.cpp +++ b/util/datetime/parser_ut.cpp @@ -22,31 +22,31 @@ Y_UNIT_TEST_SUITE(TDateTimeParseTest) { r = ParseRFC822DateTime("Fri, 4 Mar 2005 19:34:45 EST", t); UNIT_ASSERT(r); - UNIT_ASSERT_VALUES_EQUAL(t, (time_t)1109964885 + 5 * SECONDS_PER_HOUR); + UNIT_ASSERT_VALUES_EQUAL(t, (time_t)1109964885 + 5 * SECONDS_PER_HOUR); r = ParseRFC822DateTime("Fri, 4 Mar 2005 19:34:45 EDT", t); UNIT_ASSERT(r); - UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 4 * SECONDS_PER_HOUR); + UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 4 * SECONDS_PER_HOUR); r = ParseRFC822DateTime("Fri, 4 Mar 2005 19:34:45 CST", t); UNIT_ASSERT(r); - UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 6 * SECONDS_PER_HOUR); + UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 6 * SECONDS_PER_HOUR); r = ParseRFC822DateTime("Fri, 4 Mar 2005 19:34:45 CDT", t); UNIT_ASSERT(r); - UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 5 * SECONDS_PER_HOUR); + UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 5 * SECONDS_PER_HOUR); r = ParseRFC822DateTime("Fri, 4 Mar 2005 19:34:45 MST", t); UNIT_ASSERT(r); - UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 7 * SECONDS_PER_HOUR); + UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 7 * SECONDS_PER_HOUR); r = ParseRFC822DateTime("Fri, 4 Mar 2005 19:34:45 MDT", t); UNIT_ASSERT(r); - UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 6 * SECONDS_PER_HOUR); + UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 6 * SECONDS_PER_HOUR); r = ParseRFC822DateTime("Fri, 4 Mar 2005 19:34:45 PST", t); UNIT_ASSERT(r); - UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 8 * SECONDS_PER_HOUR); + UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 8 * SECONDS_PER_HOUR); r = ParseRFC822DateTime("Fri, 4 Mar 2005 19:34:45 PDT", t); UNIT_ASSERT(r); - UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 7 * SECONDS_PER_HOUR); + UNIT_ASSERT_EQUAL(t, (time_t)1109964885 + 7 * SECONDS_PER_HOUR); // optinal century r = ParseRFC822DateTime("Fri, 4 Mar 05 19:34:45 UT", t); @@ -118,14 +118,14 @@ Y_UNIT_TEST_SUITE(TDateTimeParseTest) { r = ParseRFC822DateTime(" \t Fri, \t 4 \t Mar \t 2005 \t 19:34:45 \t UT \t ", t); // spaces with tabs UNIT_ASSERT(r); UNIT_ASSERT_EQUAL(t, (time_t)1109964885); - + r = ParseRFC822DateTime("Thu, 01 Jan 1970 03:00:00 +0300", t); // spaces with tabs - UNIT_ASSERT(r); - UNIT_ASSERT_EQUAL(t, (time_t)0); - + UNIT_ASSERT(r); + UNIT_ASSERT_EQUAL(t, (time_t)0); + r = ParseRFC822DateTime("Sat, 14 Feb 2009 02:31:30 +0300", t); // spaces with tabs - UNIT_ASSERT(r); - UNIT_ASSERT_EQUAL(t, (time_t)1234567890); + UNIT_ASSERT(r); + UNIT_ASSERT_EQUAL(t, (time_t)1234567890); } time_t GetOffset(char militaryZone) { @@ -143,7 +143,7 @@ Y_UNIT_TEST_SUITE(TDateTimeParseTest) { } } - void DoTestMilitaryZones(char firstChar, char lastChar) { + void DoTestMilitaryZones(char firstChar, char lastChar) { const time_t utcTime = 1109964885; // Fri, 4 Mar 2005 19:34:45 UT char text[] = "Fri, 4 Mar 2005 19:34:45 A"; const size_t zoneCharIndex = strlen(text) - 1; @@ -155,17 +155,17 @@ Y_UNIT_TEST_SUITE(TDateTimeParseTest) { text[zoneCharIndex] = militaryZone; UNIT_ASSERT(ParseRFC822DateTime(text, t)); - UNIT_ASSERT_EQUAL(t, utcTime - offset); + UNIT_ASSERT_EQUAL(t, utcTime - offset); } } Y_UNIT_TEST(TestRfc822MilitaryZones) { - DoTestMilitaryZones('A', 'I'); - DoTestMilitaryZones('K', 'Z'); - DoTestMilitaryZones('a', 'i'); - DoTestMilitaryZones('k', 'z'); - } - + DoTestMilitaryZones('A', 'I'); + DoTestMilitaryZones('K', 'Z'); + DoTestMilitaryZones('a', 'i'); + DoTestMilitaryZones('k', 'z'); + } + Y_UNIT_TEST(TestRfc822IncorrectDates) { bool r = true; time_t t = 0; @@ -219,7 +219,7 @@ Y_UNIT_TEST_SUITE(TDateTimeParseTest) { r = ParseRFC822DateTime("17 Nov 2008 19:34:123 UT", t); UNIT_ASSERT(!r); r = ParseRFC822DateTime("17 Nov 2008 19:34:12.12 UT", t); // fractions of second are now allowed - UNIT_ASSERT(!r); + UNIT_ASSERT(!r); r = ParseRFC822DateTime("Mon , 17 Nov 2005 19:34:45 UT", t); // space after day before the comma UNIT_ASSERT(!r); @@ -254,113 +254,113 @@ Y_UNIT_TEST_SUITE(TDateTimeParseTest) { r = ParseRFC822DateTime("Mon, 17 Nov 2008 19:34:45 -03030", t); UNIT_ASSERT(!r); } - + Y_UNIT_TEST(TestRfc822Partial) { TRfc822DateTimeParser p; - const char* part1 = "Fri, 4 Mar 05 1"; - const char* part2 = "9:34:45 +0300"; - UNIT_ASSERT(p.ParsePart(part1, strlen(part1))); - UNIT_ASSERT(p.ParsePart(part2, strlen(part2))); - UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(1109954085), p.GetResult(TInstant::Max())); + const char* part1 = "Fri, 4 Mar 05 1"; + const char* part2 = "9:34:45 +0300"; + UNIT_ASSERT(p.ParsePart(part1, strlen(part1))); + UNIT_ASSERT(p.ParsePart(part2, strlen(part2))); + UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(1109954085), p.GetResult(TInstant::Max())); p = TRfc822DateTimeParser(); const char* part3 = "Fri, 4 Mar 05 19:34:46 +0300"; UNIT_ASSERT(p.ParsePart(part3, strlen(part3))); UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(1109954086), p.GetResult(TInstant::Zero())); - } + } Y_UNIT_TEST(TestIso8601Partial) { TIso8601DateTimeParser p; - const char* part1 = "1990-03-15T15:1"; - const char* part2 = "6:17+0732"; - UNIT_ASSERT(p.ParsePart(part1, strlen(part1))); - UNIT_ASSERT(p.ParsePart(part2, strlen(part2))); - UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(637487057), p.GetResult(TInstant::Max())); + const char* part1 = "1990-03-15T15:1"; + const char* part2 = "6:17+0732"; + UNIT_ASSERT(p.ParsePart(part1, strlen(part1))); + UNIT_ASSERT(p.ParsePart(part2, strlen(part2))); + UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(637487057), p.GetResult(TInstant::Max())); p = TIso8601DateTimeParser(); const char* part3 = "1990-03-15T15:16:18+0732"; UNIT_ASSERT(p.ParsePart(part3, strlen(part3))); UNIT_ASSERT_VALUES_EQUAL(TInstant::Seconds(637487058), p.GetResult(TInstant::Zero())); - } - + } + Y_UNIT_TEST(TestIso8601Correct) { - bool ret; - time_t t; - - // ISO 8601 actually does not allow time without time zone + bool ret; + time_t t; + + // ISO 8601 actually does not allow time without time zone ret = ParseISO8601DateTime("1990-03-15", t); - UNIT_ASSERT(ret); + UNIT_ASSERT(ret); UNIT_ASSERT_VALUES_EQUAL(t, 637459200); - - // some normal dates + + // some normal dates ret = ParseISO8601DateTime("1990-03-15T15:16:17Z", t); - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 637514177); - + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 637514177); + ret = ParseISO8601DateTime("1990-03-15t15:16:17z", t); // lower-case must be allowed too - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 637514177); - + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 637514177); + ret = ParseISO8601DateTime("1990-03-15 15:16:17Z", t); // space as separator should be allowed - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 637514177); - + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 637514177); + ret = ParseISO8601DateTime("1990-03-15T15:16:17.18Z", t); - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 637514177); - + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 637514177); + ret = ParseISO8601DateTime("1990-03-15T15:16:17.18+07:32", t); - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 637487057); - + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 637487057); + ret = ParseISO8601DateTime("1990-03-15T15:16:17.18+0732", t); - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 637487057); - + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 637487057); + ret = ParseISO8601DateTime("1970-01-01T00:00:00Z", t); - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 0); - + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 0); + ret = ParseISO8601DateTime("1970-01-01T00:01:02Z", t); - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 62); - -#if 0 - // these tests are disabled, because time zones are handled differently - // in old util/ parser and agalakhov@ parser + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 62); + +#if 0 + // these tests are disabled, because time zones are handled differently + // in old util/ parser and agalakhov@ parser ret = ParseISO8601DateTime("1970-01-01", t); - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, -4 * 3600); - + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, -4 * 3600); + ret = ParseISO8601DateTime("1970-01-02", t); - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 86400 - 3 * 3600); -#endif - - // this is wrong because of timezone + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 86400 - 3 * 3600); +#endif + + // this is wrong because of timezone ret = ParseISO8601DateTime("2009-02-14T03:31:30", t); - UNIT_ASSERT(ret); + UNIT_ASSERT(ret); UNIT_ASSERT_VALUES_EQUAL(t, 1234582290); - + ret = ParseISO8601DateTime("2009-02-14t03:31:30", t); - UNIT_ASSERT(ret); + UNIT_ASSERT(ret); UNIT_ASSERT_VALUES_EQUAL(t, 1234582290); - + ret = ParseISO8601DateTime("2009-02-14T02:31:30+0300", t); - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 1234567890); - + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 1234567890); + ret = ParseISO8601DateTime("2009-02-14T02:31:30+03:00", t); - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 1234567890); - + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 1234567890); + ret = ParseISO8601DateTime("2009-02-14 02:31:30+03:00", t); - UNIT_ASSERT(ret); - UNIT_ASSERT_VALUES_EQUAL(t, 1234567890); + UNIT_ASSERT(ret); + UNIT_ASSERT_VALUES_EQUAL(t, 1234567890); ret = ParseISO8601DateTime("2010-03-28T04:27:00.000-07:00", t); UNIT_ASSERT(ret); UNIT_ASSERT_VALUES_EQUAL(t, 1269775620); - } - + } + Y_UNIT_TEST(TestIso8601TimeZone) { time_t t1, t2, t3, t4; UNIT_ASSERT(ParseISO8601DateTime("2010-03-28T04:27:00.000+07:00", t1)); @@ -373,54 +373,54 @@ Y_UNIT_TEST_SUITE(TDateTimeParseTest) { } Y_UNIT_TEST(TestIso8601Incorrect) { - bool ret; - time_t t; - + bool ret; + time_t t; + t = 12345; ret = ParseISO8601DateTime("", t); UNIT_ASSERT(!ret); UNIT_ASSERT_EQUAL(t, (time_t)12345); - // some bad dates + // some bad dates t = 54321; ret = ParseISO8601DateTime("a990-01-15", t); - UNIT_ASSERT(!ret); + UNIT_ASSERT(!ret); UNIT_ASSERT_EQUAL(t, (time_t)54321); - + ret = ParseISO8601DateTime("1970-01-01T03:00:00+04:00", t); // this is 1969 GMT UNIT_ASSERT(!ret); ret = ParseISO8601DateTime("1987-13-16", t); - UNIT_ASSERT(!ret); - + UNIT_ASSERT(!ret); + ret = ParseISO8601DateTime("1987-02-29", t); - UNIT_ASSERT(!ret); - + UNIT_ASSERT(!ret); + ret = ParseISO8601DateTime("1990-03-151Y15:16:17.18", t); - UNIT_ASSERT(!ret); - + UNIT_ASSERT(!ret); + ret = ParseISO8601DateTime("1990-03-151T15:16:17:43.18", t); - UNIT_ASSERT(!ret); - + UNIT_ASSERT(!ret); + ret = ParseISO8601DateTime("1990-03-151T15:16:17.18Z+21:32", t); - UNIT_ASSERT(!ret); - } - + UNIT_ASSERT(!ret); + } + Y_UNIT_TEST(TestIso8601Fractions) { - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TInstant::ParseIso8601("2009-09-19 03:37:08.1+04:00"), TInstant::Seconds(1253317028) + TDuration::MilliSeconds(100)); - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TInstant::ParseIso8601("2009-09-19 03:37:03.926+04:00"), TInstant::Seconds(1253317023) + TDuration::MilliSeconds(926)); - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TInstant::ParseIso8601("2009-09-19 03:37:03.92622+04:00"), TInstant::Seconds(1253317023) + TDuration::MicroSeconds(926220)); - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TInstant::ParseIso8601("2009-09-19 03:37:03.012331+04:00"), TInstant::Seconds(1253317023) + TDuration::MicroSeconds(12331)); - } - + } + Y_UNIT_TEST(TestIso8601FractionsBelowMicro) { UNIT_ASSERT_VALUES_EQUAL( TInstant::ParseIso8601("1970-01-01 00:00:00.0000000+00:00"), @@ -472,26 +472,26 @@ Y_UNIT_TEST_SUITE(TDateTimeParseTest) { } Y_UNIT_TEST(TestHttpDate) { - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TInstant::ParseHttp("Sun, 06 Nov 1994 08:49:37 GMT"), TInstant::ParseIso8601("1994-11-06T08:49:37Z")); - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TInstant::ParseHttp("Sunday, 06-Nov-94 08:49:37 GMT"), TInstant::ParseIso8601("1994-11-06T08:49:37Z")); - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TInstant::ParseHttp("Sun Nov 6 08:49:37 1994"), TInstant::ParseIso8601("1994-11-06T08:49:37Z")); UNIT_ASSERT_VALUES_EQUAL( TInstant::ParseHttp("Mon, 19 Jan 2037 08:49:37 GMT"), TInstant::ParseIso8601("2037-01-19T08:49:37Z")); - } - + } + Y_UNIT_TEST(TestHttpDateIncorrect) { - bool ret; - time_t t = 0; + bool ret; + time_t t = 0; ret = ParseHTTPDateTime("1990-03-15T15:16:17Z", t); - UNIT_ASSERT(!ret); - } + UNIT_ASSERT(!ret); + } Y_UNIT_TEST(TestX509ValidityTime) { UNIT_ASSERT_VALUES_EQUAL( @@ -568,8 +568,8 @@ Y_UNIT_TEST_SUITE(TDateTimeParseTest) { UNIT_ASSERT(!TInstant::TryParseX509(s, iTry)); } } -} - +} + Y_UNIT_TEST_SUITE(TDurationParseTest) { Y_UNIT_TEST(TestParse) { UNIT_ASSERT_VALUES_EQUAL(TDuration::Seconds(60 * 60 * 24 * 7), TDuration::Parse("1w")); @@ -605,23 +605,23 @@ Y_UNIT_TEST_SUITE(TDurationParseTest) { UNIT_ASSERT_VALUES_EQUAL(TDuration::Hours(36), TDuration::Parse("1d12h")); #endif - UNIT_ASSERT_VALUES_EQUAL(TDuration::Seconds(10), TDuration::Parse("10s")); - UNIT_ASSERT_VALUES_EQUAL(TDuration::Seconds(10), TDuration::Parse("10.000s")); - UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(4), TDuration::Parse("0.000004s")); - UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(3456), TDuration::Parse("3.456s")); - UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(3450), TDuration::Parse("3.450s")); - UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(3450), TDuration::Parse("3.45000000s")); - UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(3450), TDuration::Parse("3.45s")); - - UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(1), TDuration::Parse("1ms")); - UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(1100), TDuration::Parse("1.1ms")); - - UNIT_ASSERT_VALUES_EQUAL(TDuration::Seconds(112), TDuration::Parse("112")); - UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(14456), TDuration::Parse("14456us")); + UNIT_ASSERT_VALUES_EQUAL(TDuration::Seconds(10), TDuration::Parse("10s")); + UNIT_ASSERT_VALUES_EQUAL(TDuration::Seconds(10), TDuration::Parse("10.000s")); + UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(4), TDuration::Parse("0.000004s")); + UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(3456), TDuration::Parse("3.456s")); + UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(3450), TDuration::Parse("3.450s")); + UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(3450), TDuration::Parse("3.45000000s")); + UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(3450), TDuration::Parse("3.45s")); + + UNIT_ASSERT_VALUES_EQUAL(TDuration::MilliSeconds(1), TDuration::Parse("1ms")); + UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(1100), TDuration::Parse("1.1ms")); + + UNIT_ASSERT_VALUES_EQUAL(TDuration::Seconds(112), TDuration::Parse("112")); + UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(14456), TDuration::Parse("14456us")); UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(1), TDuration::Parse("1000ns")); UNIT_ASSERT_VALUES_EQUAL(TDuration::MicroSeconds(1), TDuration::Parse("0.000001s")); UNIT_ASSERT_EQUAL(TDuration(), TDuration::Parse("10ns")); // TDuration has 1us precision. - } -} + } +} diff --git a/util/datetime/systime.cpp b/util/datetime/systime.cpp index 21f106e9da..6ee7e8fc6e 100644 --- a/util/datetime/systime.cpp +++ b/util/datetime/systime.cpp @@ -1,10 +1,10 @@ #include "systime.h" - + #include <util/system/yassert.h> #include <util/system/defaults.h> -#ifdef _win_ - +#ifdef _win_ + void FileTimeToTimeval(const FILETIME* ft, timeval* tv) { const i64 NANOINTERVAL = LL(116444736000000000); // Number of 100 nanosecond units from 1/1/1601 to 1/1/1970 union { @@ -34,7 +34,7 @@ tm* localtime_r(const time_t* clock, tm* result) { } tm* gmtime_r(const time_t* clock, tm* result) { - return gmtime_s(result, clock) == 0 ? result : 0; + return gmtime_s(result, clock) == 0 ? result : 0; } char* ctime_r(const time_t* clock, char* buf) { diff --git a/util/draft/date.cpp b/util/draft/date.cpp index 11b080820f..a290c46050 100644 --- a/util/draft/date.cpp +++ b/util/draft/date.cpp @@ -2,7 +2,7 @@ #include <util/string/cast.h> #include <util/generic/yexception.h> -#include <util/datetime/base.h> +#include <util/datetime/base.h> time_t GetDateStart(time_t ts) { tm dateTm; diff --git a/util/draft/date.h b/util/draft/date.h index 01c9ae308f..e3eb616fe5 100644 --- a/util/draft/date.h +++ b/util/draft/date.h @@ -11,9 +11,9 @@ time_t GetDateStart(time_t ts); -// Local date (without time zone) +// Local date (without time zone) class TDate { - // XXX: wrong: must store number of days since epoch + // XXX: wrong: must store number of days since epoch time_t Timestamp; public: @@ -22,7 +22,7 @@ public: { } - // XXX: wrong. Should be replace with two methods: TodayGmt() and TodayLocal() + // XXX: wrong. Should be replace with two methods: TodayGmt() and TodayLocal() static TDate Today() { return TDate(time(nullptr)); } diff --git a/util/draft/holder_vector.h b/util/draft/holder_vector.h index c03422584c..1c62055bd9 100644 --- a/util/draft/holder_vector.h +++ b/util/draft/holder_vector.h @@ -15,15 +15,15 @@ public: } ~THolderVector() { - Clear(); - } - - void Clear() { + Clear(); + } + + void Clear() { for (typename TBase::iterator it = TBase::begin(); it != TBase::end(); ++it) { if (*it) D::Destroy(*it); } - TBase::clear(); + TBase::clear(); } size_t Size() const { @@ -46,9 +46,9 @@ public: } void PushBack(THolder<T> t) { - PushBack(t.Release()); - } - + PushBack(t.Release()); + } + void Reset(size_t i, THolder<T> t) { T* current = (*this)[i]; if (current) { @@ -88,9 +88,9 @@ public: using TBase::back; using TBase::begin; using TBase::capacity; - using TBase::empty; + using TBase::empty; using TBase::end; - using TBase::front; + using TBase::front; using TBase::reserve; using TBase::size; diff --git a/util/draft/ya.make b/util/draft/ya.make index 815b771ced..e00674b682 100644 --- a/util/draft/ya.make +++ b/util/draft/ya.make @@ -10,7 +10,7 @@ IF (TSTRING_IS_STD_STRING) ENDIF() SRCS( - date.cpp + date.cpp datetime.cpp enum.cpp holder_vector.cpp diff --git a/util/folder/dirut.cpp b/util/folder/dirut.cpp index 1f22c0e261..ffc9b09f96 100644 --- a/util/folder/dirut.cpp +++ b/util/folder/dirut.cpp @@ -403,17 +403,17 @@ int mkpath(char* path, int mode) { // did not require last component of the file name to exist (other implementations will fail // if it does not). Use RealLocation if that behaviour is required. TString RealPath(const TString& path) { - TTempBuf result; + TTempBuf result; Y_ASSERT(result.Size() > MAX_PATH); //TMP_BUF_LEN > MAX_PATH -#ifdef _win_ +#ifdef _win_ if (GetFullPathName(path.data(), result.Size(), result.Data(), nullptr) == 0) -#else +#else if (realpath(path.data(), result.Data()) == nullptr) -#endif +#endif ythrow TFileError() << "RealPath failed \"" << path << "\""; - return result.Data(); -} - + return result.Data(); +} + TString RealLocation(const TString& path) { if (NFs::Exists(path)) return RealPath(path); @@ -451,37 +451,37 @@ int MakeTempDir(char path[/*FILENAME_MAX*/], const char* prefix) { bool IsDir(const TString& path) { return TFileStat(path).IsDir(); -} - +} + TString GetHomeDir() { TString s(getenv("HOME")); - if (!s) { -#ifndef _win32_ + if (!s) { +#ifndef _win32_ passwd* pw = nullptr; - s = getenv("USER"); + s = getenv("USER"); if (s) pw = getpwnam(s.data()); - else - pw = getpwuid(getuid()); - if (pw) - s = pw->pw_dir; - else -#endif - { + else + pw = getpwuid(getuid()); + if (pw) + s = pw->pw_dir; + else +#endif + { char* cur_dir = getcwd(nullptr, 0); - s = cur_dir; - free(cur_dir); - } - } - return s; -} - + s = cur_dir; + free(cur_dir); + } + } + return s; +} + void MakeDirIfNotExist(const char* path, int mode) { if (!NFs::MakeDirectory(path, NFs::EFilePermission(mode)) && !NFs::Exists(path)) { ythrow TSystemError() << "failed to create directory " << path; - } -} - + } +} + void MakePathIfNotExist(const char* path, int mode) { NFs::MakeDirectoryRecursive(path, NFs::EFilePermission(mode)); if (!NFs::Exists(path) || !TFileStat(path).IsDir()) { @@ -489,22 +489,22 @@ void MakePathIfNotExist(const char* path, int mode) { } } -const char* GetFileNameComponent(const char* f) { - const char* p = strrchr(f, LOCSLASH_C); +const char* GetFileNameComponent(const char* f) { + const char* p = strrchr(f, LOCSLASH_C); #ifdef _win_ // "/" is also valid char separator on Windows const char* p2 = strrchr(f, '/'); if (p2 > p) p = p2; #endif - - if (p) { - return p + 1; - } - - return f; -} - + + if (p) { + return p + 1; + } + + return f; +} + TString GetSystemTempDir() { #ifdef _win_ char buffer[1024]; @@ -513,19 +513,19 @@ TString GetSystemTempDir() { ythrow TSystemError() << "failed to get system temporary directory"; } return TString(buffer, size); -#else - const char* var = "TMPDIR"; - const char* def = "/tmp"; - const char* r = getenv(var); +#else + const char* var = "TMPDIR"; + const char* def = "/tmp"; + const char* r = getenv(var); const char* result = r ? r : def; return result[0] == '/' ? result : ResolveDir(result); #endif -} - +} + TString ResolveDir(const char* path) { - return ResolvePath(path, true); -} - + return ResolvePath(path, true); +} + bool SafeResolveDir(const char* path, TString& result) { try { result = ResolvePath(path, true); @@ -537,10 +537,10 @@ bool SafeResolveDir(const char* path, TString& result) { TString GetDirName(const TString& path) { return TFsPath(path).Dirname(); -} - -#ifdef _win32_ - +} + +#ifdef _win32_ + char* realpath(const char* pathname, char resolved_path[MAXPATHLEN]) { // partial implementation: no path existence check return _fullpath(resolved_path, pathname, MAXPATHLEN - 1); @@ -607,7 +607,7 @@ TString ResolvePath(const char* rel, const char* abs, bool isdir) { ythrow yexception() << "cannot resolve path: \"" << rel << "\""; return buf; } - + TString ResolvePath(const char* path, bool isDir) { return ResolvePath(path, nullptr, isDir); } diff --git a/util/folder/dirut.h b/util/folder/dirut.h index b2794d337a..2537027b12 100644 --- a/util/folder/dirut.h +++ b/util/folder/dirut.h @@ -61,7 +61,7 @@ const char* GetDirectorySeparatorS(); void RemoveDirWithContents(TString dirName); -const char* GetFileNameComponent(const char* f); +const char* GetFileNameComponent(const char* f); inline TString GetFileNameComponent(const TString& f) { return GetFileNameComponent(f.data()); @@ -70,7 +70,7 @@ inline TString GetFileNameComponent(const TString& f) { /// RealPath doesn't guarantee trailing separator to be stripped or left in place for directories. TString RealPath(const TString& path); // throws TString RealLocation(const TString& path); /// throws; last file name component doesn't need to exist - + TString GetSystemTempDir(); int MakeTempDir(char path[/*FILENAME_MAX*/], const char* prefix); diff --git a/util/folder/dirut_ut.cpp b/util/folder/dirut_ut.cpp index 2353e7a6e2..45ebfc842c 100644 --- a/util/folder/dirut_ut.cpp +++ b/util/folder/dirut_ut.cpp @@ -1,18 +1,18 @@ #include "dirut.h" #include "tempdir.h" - + #include <library/cpp/testing/unittest/registar.h> - + #include <util/generic/string.h> #include <util/memory/tempbuf.h> #include <util/stream/file.h> #include <util/system/platform.h> - + Y_UNIT_TEST_SUITE(TDirutTest) { Y_UNIT_TEST(TestRealPath) { - UNIT_ASSERT(IsDir(RealPath("."))); + UNIT_ASSERT(IsDir(RealPath("."))); } - + Y_UNIT_TEST(TestRealLocation) { UNIT_ASSERT(IsDir(RealLocation("."))); diff --git a/util/folder/fts.cpp b/util/folder/fts.cpp index 0a658d9916..0e6a6f86eb 100644 --- a/util/folder/fts.cpp +++ b/util/folder/fts.cpp @@ -39,7 +39,7 @@ #include <util/system/defaults.h> #include <util/system/error.h> -#include <stdlib.h> +#include <stdlib.h> #ifndef _win_ #include <inttypes.h> #include <sys/param.h> diff --git a/util/folder/iterator.h b/util/folder/iterator.h index cf8a621b9b..69e025b9c4 100644 --- a/util/folder/iterator.h +++ b/util/folder/iterator.h @@ -7,8 +7,8 @@ #include <util/generic/iterator.h> #include <util/generic/yexception.h> -/// Note this magic API traverses directory hierarchy - +/// Note this magic API traverses directory hierarchy + class TDirIterator: public TInputRangeAdaptor<TDirIterator> { struct TFtsDestroy { static inline void Destroy(FTS* f) noexcept { diff --git a/util/folder/iterator_ut.cpp b/util/folder/iterator_ut.cpp index 15fdda8713..936becd139 100644 --- a/util/folder/iterator_ut.cpp +++ b/util/folder/iterator_ut.cpp @@ -10,13 +10,13 @@ #include <util/random/mersenne.h> static TString JoinWithNewline(const TVector<TString>& strings) { - TStringStream ss; + TStringStream ss; for (const auto& string : strings) { ss << string << "\n"; - } - return ss.Str(); -} - + } + return ss.Str(); +} + class TDirIteratorTest: public TTestBase { UNIT_TEST_SUITE(TDirIteratorTest); UNIT_TEST(TestIt) diff --git a/util/folder/path.cpp b/util/folder/path.cpp index 60a14b3fe7..bfe0c67d68 100644 --- a/util/folder/path.cpp +++ b/util/folder/path.cpp @@ -7,14 +7,14 @@ #include <util/system/compiler.h> #include <util/system/file.h> #include <util/system/fs.h> - + struct TFsPath::TSplit: public TAtomicRefCount<TSplit>, public TPathSplit { inline TSplit(const TStringBuf path) : TPathSplit(path) { } -}; - +}; + void TFsPath::CheckDefined() const { if (!IsDefined()) { ythrow TIoException() << TStringBuf("must be defined"); @@ -143,62 +143,62 @@ TString TFsPath::GetName() const { if (!IsDefined()) { return TString(); } - + const TSplit& split = GetSplit(); if (split.size() > 0) { if (split.back() != "..") { return TString(split.back()); - } else { - // cannot just drop last component, because path itself may be a symlink - return RealPath().GetName(); - } - } else { + } else { + // cannot just drop last component, because path itself may be a symlink + return RealPath().GetName(); + } + } else { if (split.IsAbsolute) { - return split.Reconstruct(); - } else { - return Cwd().GetName(); - } - } -} - + return split.Reconstruct(); + } else { + return Cwd().GetName(); + } + } +} + TString TFsPath::GetExtension() const { return TString(GetSplit().Extension()); } -bool TFsPath::IsAbsolute() const { +bool TFsPath::IsAbsolute() const { return GetSplit().IsAbsolute; -} - -bool TFsPath::IsRelative() const { - return !IsAbsolute(); -} - -void TFsPath::InitSplit() const { +} + +bool TFsPath::IsRelative() const { + return !IsAbsolute(); +} + +void TFsPath::InitSplit() const { Split_ = new TSplit(Path_); -} - -TFsPath::TSplit& TFsPath::GetSplit() const { - // XXX: race condition here +} + +TFsPath::TSplit& TFsPath::GetSplit() const { + // XXX: race condition here if (!Split_) { - InitSplit(); + InitSplit(); } return *Split_; -} - +} + static Y_FORCE_INLINE void VerifyPath(const TStringBuf path) { Y_VERIFY(!path.Contains('\0'), "wrong format of TFsPath"); } -TFsPath::TFsPath() { -} - +TFsPath::TFsPath() { +} + TFsPath::TFsPath(const TString& path) : Path_(path) { VerifyPath(Path_); -} - +} + TFsPath::TFsPath(const TStringBuf path) : Path_(ToString(path)) { @@ -208,56 +208,56 @@ TFsPath::TFsPath(const TStringBuf path) TFsPath::TFsPath(const char* path) : Path_(path) { -} - +} + TFsPath TFsPath::Child(const TString& name) const { if (!name) { ythrow TIoException() << "child name must not be empty"; } return *this / name; -} - -struct TClosedir { - static void Destroy(DIR* dir) { +} + +struct TClosedir { + static void Destroy(DIR* dir) { if (dir) { if (0 != closedir(dir)) { ythrow TIoSystemError() << "failed to closedir"; } } - } -}; - + } +}; + void TFsPath::ListNames(TVector<TString>& children) const { - CheckDefined(); + CheckDefined(); THolder<DIR, TClosedir> dir(opendir(this->c_str())); if (!dir) { ythrow TIoSystemError() << "failed to opendir " << Path_; } - for (;;) { - struct dirent de; - struct dirent* ok; + for (;;) { + struct dirent de; + struct dirent* ok; // TODO(yazevnul|IGNIETFERRO-1070): remove these macroses by replacing `readdir_r` with proper // alternative Y_PRAGMA_DIAGNOSTIC_PUSH Y_PRAGMA_NO_DEPRECATED - int r = readdir_r(dir.Get(), &de, &ok); + int r = readdir_r(dir.Get(), &de, &ok); Y_PRAGMA_DIAGNOSTIC_POP if (r != 0) { ythrow TIoSystemError() << "failed to readdir " << Path_; } if (ok == nullptr) { - return; + return; } TString name(de.d_name); if (name == "." || name == "..") { - continue; + continue; } - children.push_back(name); - } -} - + children.push_back(name); + } +} + bool TFsPath::Contains(const TString& component) const { if (!IsDefined()) { return false; @@ -277,43 +277,43 @@ bool TFsPath::Contains(const TString& component) const { void TFsPath::List(TVector<TFsPath>& files) const { TVector<TString> names; - ListNames(names); + ListNames(names); for (auto& name : names) { files.push_back(Child(name)); - } -} - + } +} + void TFsPath::RenameTo(const TString& newPath) const { - CheckDefined(); + CheckDefined(); if (!newPath) { - ythrow TIoException() << "bad new file name"; + ythrow TIoException() << "bad new file name"; } if (!NFs::Rename(Path_, newPath)) { ythrow TIoSystemError() << "failed to rename " << Path_ << " to " << newPath; } -} - -void TFsPath::RenameTo(const char* newPath) const { +} + +void TFsPath::RenameTo(const char* newPath) const { RenameTo(TString(newPath)); -} - -void TFsPath::RenameTo(const TFsPath& newPath) const { - RenameTo(newPath.GetPath()); -} - -void TFsPath::Touch() const { - CheckDefined(); - if (!TFile(*this, OpenAlways).IsOpen()) { - ythrow TIoException() << "failed to touch " << *this; - } -} - -// XXX: move implementation to util/somewhere. -TFsPath TFsPath::RealPath() const { - CheckDefined(); +} + +void TFsPath::RenameTo(const TFsPath& newPath) const { + RenameTo(newPath.GetPath()); +} + +void TFsPath::Touch() const { + CheckDefined(); + if (!TFile(*this, OpenAlways).IsOpen()) { + ythrow TIoException() << "failed to touch " << *this; + } +} + +// XXX: move implementation to util/somewhere. +TFsPath TFsPath::RealPath() const { + CheckDefined(); return ::RealPath(*this); -} - +} + TFsPath TFsPath::RealLocation() const { CheckDefined(); return ::RealLocation(*this); @@ -329,20 +329,20 @@ TFsPath TFsPath::ReadLink() const { return NFs::ReadLink(*this); } -bool TFsPath::Exists() const { +bool TFsPath::Exists() const { return IsDefined() && NFs::Exists(*this); -} - +} + void TFsPath::CheckExists() const { if (!Exists()) { ythrow TIoException() << "path does not exist " << Path_; } } -bool TFsPath::IsDirectory() const { +bool TFsPath::IsDirectory() const { return IsDefined() && TFileStat(GetPath().data()).IsDir(); -} - +} + bool TFsPath::IsFile() const { return IsDefined() && TFileStat(GetPath().data()).IsFile(); } @@ -351,20 +351,20 @@ bool TFsPath::IsSymlink() const { return IsDefined() && TFileStat(GetPath().data(), true).IsSymlink(); } -void TFsPath::DeleteIfExists() const { +void TFsPath::DeleteIfExists() const { if (!IsDefined()) { return; } ::unlink(this->c_str()); ::rmdir(this->c_str()); - if (Exists()) { + if (Exists()) { ythrow TIoException() << "failed to delete " << Path_; - } -} - + } +} + void TFsPath::MkDir(const int mode) const { - CheckDefined(); + CheckDefined(); if (!Exists()) { int r = Mkdir(this->c_str(), mode); if (r != 0) { @@ -376,8 +376,8 @@ void TFsPath::MkDir(const int mode) const { } } } -} - +} + void TFsPath::MkDirs(const int mode) const { CheckDefined(); if (!Exists()) { @@ -386,7 +386,7 @@ void TFsPath::MkDirs(const int mode) const { } } -void TFsPath::ForceDelete() const { +void TFsPath::ForceDelete() const { if (!IsDefined()) { return; } @@ -408,20 +408,20 @@ void TFsPath::ForceDelete() const { ClearLastSystemError(); if (stat.IsDir()) { TVector<TFsPath> children; - List(children); + List(children); for (auto& i : children) { i.ForceDelete(); - } + } ::rmdir(this->c_str()); } else { ::unlink(this->c_str()); - } + } if (LastSystemError()) { ythrow TIoException() << "failed to delete " << Path_; } -} - +} + void TFsPath::CopyTo(const TString& newPath, bool force) const { if (IsDirectory()) { if (force) { @@ -458,18 +458,18 @@ void TFsPath::ForceRenameTo(const TString& newPath) const { } } -TFsPath TFsPath::Cwd() { +TFsPath TFsPath::Cwd() { return TFsPath(::NFs::CurrentWorkingDirectory()); -} - +} + const TPathSplit& TFsPath::PathSplit() const { return GetSplit(); } template <> void Out<TFsPath>(IOutputStream& os, const TFsPath& f) { - os << f.GetPath(); -} + os << f.GetPath(); +} template <> TFsPath FromStringImpl<TFsPath>(const char* s, size_t len) { diff --git a/util/folder/path.h b/util/folder/path.h index 01d41df1d2..2fb4d6b4ef 100644 --- a/util/folder/path.h +++ b/util/folder/path.h @@ -1,5 +1,5 @@ #pragma once - + #include "fwd.h" #include "pathsplit.h" @@ -9,7 +9,7 @@ #include <util/generic/vector.h> #include <util/string/cast.h> #include <util/system/fstat.h> -#include <util/system/platform.h> +#include <util/system/platform.h> #include <util/system/sysstat.h> #include <util/system/yassert.h> @@ -19,26 +19,26 @@ * Class behaviour is platform-dependent. * It uses platform-dependent separators for path-reconstructing operations. */ -class TFsPath { +class TFsPath { private: - struct TSplit; - -public: - TFsPath(); + struct TSplit; + +public: + TFsPath(); TFsPath(const TString& path); TFsPath(const TStringBuf path); - TFsPath(const char* path); - + TFsPath(const char* path); + TFsPath(const std::string& path) : TFsPath(TStringBuf(path)) { } void CheckDefined() const; - - inline bool IsDefined() const { + + inline bool IsDefined() const { return Path_.length() > 0; - } + } inline explicit operator bool() const { return IsDefined(); @@ -50,16 +50,16 @@ public: inline operator const TString&() const { return Path_; - } + } - inline bool operator==(const TFsPath& that) const { + inline bool operator==(const TFsPath& that) const { return Path_ == that.Path_; - } + } inline bool operator!=(const TFsPath& that) const { return Path_ != that.Path_; } - + TFsPath& operator/=(const TFsPath& that); friend TFsPath operator/(const TFsPath& s, const TFsPath& p) { @@ -73,11 +73,11 @@ public: inline const TString& GetPath() const { return Path_; - } - - /// last component of path, or "/" if root + } + + /// last component of path, or "/" if root TString GetName() const; - + /** * "a.b.tmp" -> "tmp" * "a.tmp" -> "tmp" @@ -85,8 +85,8 @@ public: */ TString GetExtension() const; - bool IsAbsolute() const; - bool IsRelative() const; + bool IsAbsolute() const; + bool IsRelative() const; /** * TFsPath("/a/b").IsSubpathOf("/a") -> true @@ -113,7 +113,7 @@ public: bool IsContainerOf(const TFsPath& that) const { return that.IsSubpathOf(*this); } - + TFsPath RelativeTo(const TFsPath& root) const; //must be subpath of root /** @@ -121,10 +121,10 @@ public: */ TFsPath RelativePath(const TFsPath& root) const; //..; for relative paths 1st component must be the same - /** - * Never fails. Returns this if already a root. - */ - TFsPath Parent() const; + /** + * Never fails. Returns this if already a root. + */ + TFsPath Parent() const; TString Basename() const { return GetName(); @@ -132,9 +132,9 @@ public: TString Dirname() const { return Parent(); } - + TFsPath Child(const TString& name) const; - + /** * @brief create this directory * @@ -143,7 +143,7 @@ public: * Nothing to do if dir exists. */ void MkDir(const int mode = MODE0777) const; - + /** * @brief create this directory and all parent directories as needed * @@ -151,66 +151,66 @@ public: */ void MkDirs(const int mode = MODE0777) const; - // XXX: rewrite to return iterator + // XXX: rewrite to return iterator void List(TVector<TFsPath>& children) const; void ListNames(TVector<TString>& children) const; - + // Check, if path contains at least one component with a specific name. bool Contains(const TString& component) const; - // fails to delete non-empty directory - void DeleteIfExists() const; - // delete recursively. Does nothing if not exists - void ForceDelete() const; - - // XXX: ino - + // fails to delete non-empty directory + void DeleteIfExists() const; + // delete recursively. Does nothing if not exists + void ForceDelete() const; + + // XXX: ino + inline bool Stat(TFileStat& stat) const { stat = TFileStat(Path_.data()); return stat.Mode; } - bool Exists() const; - /// false if not exists - bool IsDirectory() const; + bool Exists() const; + /// false if not exists + bool IsDirectory() const; /// false if not exists bool IsFile() const; /// false if not exists - bool IsSymlink() const; - /// throw TIoException if not exists - void CheckExists() const; - + bool IsSymlink() const; + /// throw TIoException if not exists + void CheckExists() const; + void RenameTo(const TString& newPath) const; - void RenameTo(const char* newPath) const; - void RenameTo(const TFsPath& newFile) const; + void RenameTo(const char* newPath) const; + void RenameTo(const TFsPath& newFile) const; void ForceRenameTo(const TString& newPath) const; - + void CopyTo(const TString& newPath, bool force) const; - void Touch() const; - - TFsPath RealPath() const; + void Touch() const; + + TFsPath RealPath() const; TFsPath RealLocation() const; TFsPath ReadLink() const; - - /// always absolute - static TFsPath Cwd(); - + + /// always absolute + static TFsPath Cwd(); + inline void Swap(TFsPath& p) noexcept { DoSwap(Path_, p.Path_); Split_.Swap(p.Split_); } -private: - void InitSplit() const; - TSplit& GetSplit() const; +private: + void InitSplit() const; + TSplit& GetSplit() const; private: TString Path_; /// cache mutable TSimpleIntrusivePtr<TSplit> Split_; -}; +}; namespace NPrivate { inline void AppendToFsPath(TFsPath&) { diff --git a/util/folder/path_ut.cpp b/util/folder/path_ut.cpp index 81ecf0e034..e6a3451016 100644 --- a/util/folder/path_ut.cpp +++ b/util/folder/path_ut.cpp @@ -6,12 +6,12 @@ #include <library/cpp/testing/unittest/registar.h> #include <util/generic/scope.h> -#include <util/system/platform.h> +#include <util/system/platform.h> #include <util/system/yassert.h> -#include <util/stream/output.h> +#include <util/stream/output.h> #include <util/stream/file.h> #include <util/system/fs.h> - + #include <algorithm> #ifdef _win_ @@ -116,19 +116,19 @@ Y_UNIT_TEST_SUITE(TFsPathTests) { UNIT_ASSERT_VALUES_EQUAL(TFsPath("..").Parent(), TFsPath("../..")); #endif } - + Y_UNIT_TEST(GetName) { TTestDirectory d("GetName"); UNIT_ASSERT_VALUES_EQUAL(TString("dfgh"), d.Child("dfgh").GetName()); - + // check does not fail TFsPath(".").GetName(); - -#ifdef _unix_ + +#ifdef _unix_ UNIT_ASSERT_VALUES_EQUAL(TString("/"), TFsPath("/").GetName()); -#endif +#endif } - + Y_UNIT_TEST(GetExtension) { TTestDirectory d("GetExtension"); UNIT_ASSERT_VALUES_EQUAL("", d.Child("a").GetExtension()); @@ -147,7 +147,7 @@ Y_UNIT_TEST_SUITE(TFsPathTests) { UNIT_ASSERT(!f1.Exists()); UNIT_ASSERT(f2.Exists()); } - + Y_UNIT_TEST(TestForceRename) { TTestDirectory xx("TestForceRename"); TFsPath fMain = xx.Child("main"); @@ -171,7 +171,7 @@ Y_UNIT_TEST_SUITE(TFsPathTests) { Y_UNIT_TEST(TestRenameFail) { UNIT_ASSERT_EXCEPTION(TFsPath("sfsfsfsdfsfsdfdf").RenameTo("sdfsdf"), TIoException); } - + #ifndef _win_ Y_UNIT_TEST(TestRealPath) { UNIT_ASSERT(TFsPath(".").RealPath().IsDirectory()); @@ -189,7 +189,7 @@ Y_UNIT_TEST_SUITE(TFsPathTests) { UNIT_ASSERT_VALUES_EQUAL(link.RealPath(), target2.RealPath()); // must not cache old value } #endif - + Y_UNIT_TEST(TestSlashesAndBasename) { TFsPath p("/db/BASE/primus121-025-1380131338//"); UNIT_ASSERT_VALUES_EQUAL(p.Basename(), TString("primus121-025-1380131338")); @@ -227,27 +227,27 @@ Y_UNIT_TEST_SUITE(TFsPathTests) { Y_UNIT_TEST(TestList) { TTestDirectory td("TestList-dir"); - + TFsPath dir = td.GetFsPath(); dir.Child("a").Touch(); dir.Child("b").MkDir(); dir.Child("b").Child("b-1").Touch(); dir.Child("c").MkDir(); dir.Child("d").Touch(); - + TVector<TString> children; dir.ListNames(children); std::sort(children.begin(), children.end()); - + TVector<TString> expected; expected.push_back("a"); expected.push_back("b"); expected.push_back("c"); expected.push_back("d"); - + UNIT_ASSERT_VALUES_EQUAL(expected, children); } - + #ifdef _unix_ Y_UNIT_TEST(MkDirMode) { TTestDirectory td("MkDirMode"); @@ -809,4 +809,4 @@ Y_UNIT_TEST_SUITE(TFsPathTests) { UNIT_ASSERT_EXCEPTION_CONTAINS(testSymlink.ForceDelete(), TIoException, "failed to delete"); } #endif -} +} diff --git a/util/generic/bitops.h b/util/generic/bitops.h index fb12548d12..2db15fc59b 100644 --- a/util/generic/bitops.h +++ b/util/generic/bitops.h @@ -62,7 +62,7 @@ namespace NBitOps { Y_ASSERT(value); // because __builtin_clz* have undefined result for zero. return std::numeric_limits<unsigned int>::digits - __builtin_clz(value); } - + inline unsigned GetValueBitCountImpl(unsigned long value) noexcept { Y_ASSERT(value); // because __builtin_clz* have undefined result for zero. return std::numeric_limits<unsigned long>::digits - __builtin_clzl(value); diff --git a/util/generic/buffer.h b/util/generic/buffer.h index d3cbf95a01..9576467404 100644 --- a/util/generic/buffer.h +++ b/util/generic/buffer.h @@ -87,7 +87,7 @@ public: inline explicit operator bool() const noexcept { return Size(); } - + inline size_t Avail() const noexcept { return Len_ - Pos_; } @@ -123,9 +123,9 @@ public: } else { memmove(Data_ + pos, Data_ + end, Pos_ - end); Pos_ -= count; - } + } } - + inline void ChopHead(size_t count) { Chop(0U, count); } @@ -136,7 +136,7 @@ public: } inline void Advance(size_t len) { - Resize(Pos_ + len); + Resize(Pos_ + len); } inline void Reserve(size_t len) { diff --git a/util/generic/cast.h b/util/generic/cast.h index 1868a7a36b..0d4a41f385 100644 --- a/util/generic/cast.h +++ b/util/generic/cast.h @@ -10,37 +10,37 @@ #include <cstdlib> -template <class T, class F> -static inline T VerifyDynamicCast(F f) { - if (!f) { +template <class T, class F> +static inline T VerifyDynamicCast(F f) { + if (!f) { return nullptr; - } - - T ret = dynamic_cast<T>(f); - + } + + T ret = dynamic_cast<T>(f); + Y_VERIFY(ret, "verify cast failed"); - - return ret; -} - + + return ret; +} + #if !defined(NDEBUG) #define USE_DEBUG_CHECKED_CAST #endif -namespace NPrivate { - template <typename T, typename F> - static T DynamicCast(F f) { - return dynamic_cast<T>(f); - } -} - +namespace NPrivate { + template <typename T, typename F> + static T DynamicCast(F f) { + return dynamic_cast<T>(f); + } +} + /* * replacement for dynamic_cast(dynamic_cast in debug mode, else static_cast) */ template <class T, class F> static inline T CheckedCast(F f) { #if defined(USE_DEBUG_CHECKED_CAST) - return VerifyDynamicCast<T>(f); + return VerifyDynamicCast<T>(f); #else /* Make sure F is polymorphic. * Without this cast, CheckedCast with non-polymorphic F diff --git a/util/generic/maybe.h b/util/generic/maybe.h index b07de3b3b5..34d21aebcd 100644 --- a/util/generic/maybe.h +++ b/util/generic/maybe.h @@ -222,7 +222,7 @@ public: } ~TMaybe() = default; - + constexpr TMaybe& operator=(const TMaybe&) = default; constexpr TMaybe& operator=(TMaybe&&) = default; @@ -238,9 +238,9 @@ public: } else { Init(std::forward<U>(right)); } - return *this; - } - + return *this; + } + template <class U> std::enable_if_t<TCopyAssignable<U>::value, TMaybe&> @@ -289,16 +289,16 @@ public: this->Defined_ = false; Data()->~T(); } - } - + } + constexpr bool Defined() const noexcept { return this->Defined_; } - + Y_PURE_FUNCTION constexpr bool Empty() const noexcept { return !Defined(); - } - + } + void CheckDefined() const { if (Y_UNLIKELY(!Defined())) { Policy::OnEmpty(typeid(TValueType)); @@ -314,17 +314,17 @@ public: } constexpr const T& GetRef() const& { - CheckDefined(); + CheckDefined(); return *Data(); - } - + } + constexpr T& GetRef() & { - CheckDefined(); + CheckDefined(); return *Data(); - } - + } + constexpr const T&& GetRef() const&& { CheckDefined(); @@ -340,11 +340,11 @@ public: constexpr const T& operator*() const& { return GetRef(); } - + constexpr T& operator*() & { return GetRef(); - } - + } + constexpr const T&& operator*() const&& { return std::move(GetRef()); } @@ -355,8 +355,8 @@ public: constexpr const T* operator->() const { return &GetRef(); - } - + } + constexpr T* operator->() { return &GetRef(); } @@ -384,8 +384,8 @@ public: constexpr explicit operator bool() const noexcept { return Defined(); - } - + } + void Save(IOutputStream* out) const { const bool defined = Defined(); diff --git a/util/generic/maybe_ut.cpp b/util/generic/maybe_ut.cpp index a9afc4661a..2c1a425c5e 100644 --- a/util/generic/maybe_ut.cpp +++ b/util/generic/maybe_ut.cpp @@ -2,24 +2,24 @@ #include <util/generic/vector.h> #include <util/stream/str.h> #include <library/cpp/testing/unittest/registar.h> - -#include "maybe.h" - -class TIncrementOnDestroy { -private: + +#include "maybe.h" + +class TIncrementOnDestroy { +private: int* Ptr_; -public: +public: TIncrementOnDestroy(int* ptr) noexcept : Ptr_(ptr) { - } + } ~TIncrementOnDestroy() { ++*Ptr_; } -}; - +}; + Y_UNIT_TEST_SUITE(TMaybeTest) { Y_UNIT_TEST(TestStatic) { using T1 = TMaybe<int>; @@ -56,35 +56,35 @@ Y_UNIT_TEST_SUITE(TMaybeTest) { Y_UNIT_TEST(TTestConstructorDestructor) { int a = 0; int b = 0; - - TMaybe<TIncrementOnDestroy>(); + + TMaybe<TIncrementOnDestroy>(); UNIT_ASSERT_VALUES_EQUAL(a, b); - + TMaybe<TIncrementOnDestroy>(TIncrementOnDestroy(&a)); b += 2; UNIT_ASSERT_VALUES_EQUAL(a, b); - - { + + { TMaybe<TIncrementOnDestroy> m1 = TIncrementOnDestroy(&a); b += 1; UNIT_ASSERT_VALUES_EQUAL(a, b); - + TMaybe<TIncrementOnDestroy> m2 = m1; UNIT_ASSERT_VALUES_EQUAL(a, b); - + TMaybe<TIncrementOnDestroy> m3; m3 = m1; UNIT_ASSERT_VALUES_EQUAL(a, b); - } - + } + b += 3; UNIT_ASSERT_VALUES_EQUAL(a, b); - - { + + { TMaybe<TIncrementOnDestroy> m4 = TIncrementOnDestroy(&a); b += 1; UNIT_ASSERT_VALUES_EQUAL(a, b); - + m4 = TIncrementOnDestroy(&a); b += 1; UNIT_ASSERT_VALUES_EQUAL(a, b); @@ -95,9 +95,9 @@ Y_UNIT_TEST_SUITE(TMaybeTest) { m4.Clear(); UNIT_ASSERT_VALUES_EQUAL(a, b); - } - } - + } + } + Y_UNIT_TEST(TestAssignmentClear) { TMaybe<int> m5; UNIT_ASSERT(!m5.Defined()); @@ -105,24 +105,24 @@ Y_UNIT_TEST_SUITE(TMaybeTest) { UNIT_ASSERT(m5 == TMaybe<int>()); UNIT_ASSERT(m5 == Nothing()); UNIT_ASSERT(m5 != TMaybe<int>(4)); - + m5 = 4; - + UNIT_ASSERT(m5.Defined()); UNIT_ASSERT(!m5.Empty()); - + UNIT_ASSERT_VALUES_EQUAL(4, m5.GetRef()); UNIT_ASSERT(m5 == TMaybe<int>(4)); UNIT_ASSERT(m5 != TMaybe<int>(3)); UNIT_ASSERT(m5 != TMaybe<int>()); UNIT_ASSERT(m5 != Nothing()); - + m5 = TMaybe<int>(5); UNIT_ASSERT(m5.Defined()); UNIT_ASSERT_VALUES_EQUAL(5, m5.GetRef()); UNIT_ASSERT(m5 == TMaybe<int>(5)); UNIT_ASSERT(m5 != TMaybe<int>(4)); - + m5 = TMaybe<int>(); UNIT_ASSERT(m5.Empty()); UNIT_ASSERT(m5 == TMaybe<int>()); @@ -139,7 +139,7 @@ Y_UNIT_TEST_SUITE(TMaybeTest) { m5 = {}; UNIT_ASSERT(m5.Empty()); - } + } Y_UNIT_TEST(TestInPlace) { TMaybe<int> m; @@ -1003,4 +1003,4 @@ Y_UNIT_TEST_SUITE(TMaybeTest) { TMaybe<TStringBuf> v; UNIT_ASSERT_EXCEPTION_CONTAINS(v.GetRef(), yexception, "StringBuf"); } -} +} diff --git a/util/generic/object_counter.h b/util/generic/object_counter.h index 71ec5d57c4..5257afa2e6 100644 --- a/util/generic/object_counter.h +++ b/util/generic/object_counter.h @@ -1,7 +1,7 @@ -#pragma once - -#include <util/system/atomic.h> - +#pragma once + +#include <util/system/atomic.h> + /** * Simple thread-safe per-class counter that can be used to make sure you don't * have any leaks in your code, or for statistical purposes. @@ -16,13 +16,13 @@ * Cerr << "TMyClass instances in use: " << TMyClass::ObjectCount() << Endl; * \endcode */ -template <class T> -class TObjectCounter { +template <class T> +class TObjectCounter { public: inline TObjectCounter() noexcept { AtomicIncrement(Count_); } - + inline TObjectCounter(const TObjectCounter& /*item*/) noexcept { AtomicIncrement(Count_); } @@ -30,11 +30,11 @@ public: inline ~TObjectCounter() { AtomicDecrement(Count_); } - + static inline long ObjectCount() noexcept { return AtomicGet(Count_); } - + /** * Resets object count. Mainly for tests, as you don't want to do this in * your code and then end up with negative counts. @@ -44,10 +44,10 @@ public: static inline long ResetObjectCount() noexcept { return AtomicSwap(&Count_, 0); } - + private: static TAtomic Count_; -}; - -template <class T> -TAtomic TObjectCounter<T>::Count_ = 0; +}; + +template <class T> +TAtomic TObjectCounter<T>::Count_ = 0; diff --git a/util/generic/ptr.h b/util/generic/ptr.h index 3b02d9c19f..19db0e3ec5 100644 --- a/util/generic/ptr.h +++ b/util/generic/ptr.h @@ -370,7 +370,7 @@ public: Y_ASSERT(resultCount >= d); (void)resultCount; } - + inline void Ref() noexcept { auto resultCount = Counter_.Inc(); Y_ASSERT(resultCount != 0); @@ -388,7 +388,7 @@ public: inline void UnRef() noexcept { UnRef(1); } - + inline TAtomicBase RefCount() const noexcept { return Counter_.Val(); } @@ -876,7 +876,7 @@ public: inline void Drop() noexcept { TSharedPtr().Swap(*this); } - + inline T* Get() const noexcept { return T_; } diff --git a/util/generic/refcount.h b/util/generic/refcount.h index c567cdfa3a..966e853b77 100644 --- a/util/generic/refcount.h +++ b/util/generic/refcount.h @@ -23,11 +23,11 @@ public: Check(); return Counter_ += d; } - + inline TAtomicBase Inc() noexcept { return Add(1); } - + inline TAtomicBase Sub(TAtomicBase d) noexcept { Check(); return Counter_ -= d; @@ -112,7 +112,7 @@ public: inline TAtomicBase Add(TAtomicBase d) noexcept { return AtomicAdd(Counter_, d); } - + inline TAtomicBase Inc() noexcept { return Add(1); } @@ -120,7 +120,7 @@ public: inline TAtomicBase Sub(TAtomicBase d) noexcept { return AtomicSub(Counter_, d); } - + inline TAtomicBase Dec() noexcept { return Sub(1); } diff --git a/util/generic/string.h b/util/generic/string.h index f3e28037e6..8cd8aa6917 100644 --- a/util/generic/string.h +++ b/util/generic/string.h @@ -1133,7 +1133,7 @@ public: extern TBasicString EscapeC(const TBasicString&); return TBasicString() + '"' + EscapeC(*this) + '"'; - } + } /** * Modifies the case of the string, depending on the operation. diff --git a/util/generic/typetraits_ut.cpp b/util/generic/typetraits_ut.cpp index 59b6f2a9cf..e7571c75ec 100644 --- a/util/generic/typetraits_ut.cpp +++ b/util/generic/typetraits_ut.cpp @@ -5,17 +5,17 @@ #include <vector> #include <tuple> -namespace { - enum ETestEnum { - }; - +namespace { + enum ETestEnum { + }; + class TPodClass { - }; - + }; + class TNonPodClass { TNonPodClass() { } - }; + }; class TEmptyClass { void operator()() const { @@ -70,8 +70,8 @@ namespace { public: TEmptyClass Base; }; -} - +} + #define ASSERT_SAME_TYPE(x, y) \ { \ const bool x_ = std::is_same<x, y>::value; \ @@ -259,82 +259,82 @@ Y_UNIT_TEST_SUITE(TTypeTraitsTest) { } }; -namespace { - template <typename T> - struct TTypeTraitsExpected; - - template <> - struct TTypeTraitsExpected<void> { - enum { IsIntegral = false }; - enum { IsArithmetic = false }; - enum { IsPod = true }; - enum { IsVolatile = false }; - enum { IsConstant = false }; - enum { IsPointer = false }; - enum { IsReference = false }; +namespace { + template <typename T> + struct TTypeTraitsExpected; + + template <> + struct TTypeTraitsExpected<void> { + enum { IsIntegral = false }; + enum { IsArithmetic = false }; + enum { IsPod = true }; + enum { IsVolatile = false }; + enum { IsConstant = false }; + enum { IsPointer = false }; + enum { IsReference = false }; enum { IsLvalueReference = false }; enum { IsRvalueReference = false }; - enum { IsArray = false }; - enum { IsClassType = false }; - enum { IsVoid = true }; - enum { IsEnum = false }; - }; - - template <> - struct TTypeTraitsExpected<int>: public TTypeTraitsExpected<void> { - enum { IsIntegral = true }; - enum { IsArithmetic = true }; - enum { IsVoid = false }; - }; - - template <> + enum { IsArray = false }; + enum { IsClassType = false }; + enum { IsVoid = true }; + enum { IsEnum = false }; + }; + + template <> + struct TTypeTraitsExpected<int>: public TTypeTraitsExpected<void> { + enum { IsIntegral = true }; + enum { IsArithmetic = true }; + enum { IsVoid = false }; + }; + + template <> struct TTypeTraitsExpected<size_t>: public TTypeTraitsExpected<int> { - }; - - template <> + }; + + template <> struct TTypeTraitsExpected<float>: public TTypeTraitsExpected<int> { - enum { IsIntegral = false }; - }; - - template <> + enum { IsIntegral = false }; + }; + + template <> struct TTypeTraitsExpected<long double>: public TTypeTraitsExpected<float> { - }; - - template <> + }; + + template <> struct TTypeTraitsExpected<const int>: public TTypeTraitsExpected<int> { - enum { IsConstant = true }; - }; - - template <> + enum { IsConstant = true }; + }; + + template <> struct TTypeTraitsExpected<volatile int>: public TTypeTraitsExpected<int> { - enum { IsVolatile = true }; - }; - - template <> + enum { IsVolatile = true }; + }; + + template <> struct TTypeTraitsExpected<ETestEnum>: public TTypeTraitsExpected<int> { - enum { IsIntegral = false }; - enum { IsArithmetic = false }; - enum { IsEnum = true }; - }; - - template <> + enum { IsIntegral = false }; + enum { IsArithmetic = false }; + enum { IsEnum = true }; + }; + + template <> struct TTypeTraitsExpected<TPodClass>: public TTypeTraitsExpected<void> { - enum { IsClassType = true }; - enum { IsVoid = false }; - }; - - template <> + enum { IsClassType = true }; + enum { IsVoid = false }; + }; + + template <> struct TTypeTraitsExpected<TNonPodClass>: public TTypeTraitsExpected<TPodClass> { - enum { IsPod = false }; - }; - - template <> + enum { IsPod = false }; + }; + + template <> struct TTypeTraitsExpected<TNonPodClass&>: public TTypeTraitsExpected<TNonPodClass> { - enum { IsClassType = false }; - enum { IsReference = true }; + enum { IsClassType = false }; + enum { IsReference = true }; enum { IsLvalueReference = true }; - }; - + }; + template <> struct TTypeTraitsExpected<TNonPodClass&&>: public TTypeTraitsExpected<TNonPodClass> { enum { IsClassType = false }; @@ -342,25 +342,25 @@ namespace { enum { IsRvalueReference = true }; }; - template <> + template <> struct TTypeTraitsExpected<const TNonPodClass&>: public TTypeTraitsExpected<TNonPodClass&> { - }; - - template <> + }; + + template <> struct TTypeTraitsExpected<float*>: public TTypeTraitsExpected<int> { - enum { IsIntegral = false }; - enum { IsArithmetic = false }; - enum { IsPointer = true }; - }; - - template <> + enum { IsIntegral = false }; + enum { IsArithmetic = false }; + enum { IsPointer = true }; + }; + + template <> struct TTypeTraitsExpected<float&>: public TTypeTraitsExpected<float*> { - enum { IsPointer = false }; - enum { IsReference = true }; + enum { IsPointer = false }; + enum { IsReference = true }; enum { IsLvalueReference = true }; - }; - - template <> + }; + + template <> struct TTypeTraitsExpected<float&&>: public TTypeTraitsExpected<float*> { enum { IsPointer = false }; enum { IsReference = true }; @@ -369,22 +369,22 @@ namespace { template <> struct TTypeTraitsExpected<const float&>: public TTypeTraitsExpected<float&> { - }; - - template <> + }; + + template <> struct TTypeTraitsExpected<float[17]>: public TTypeTraitsExpected<int> { - enum { IsIntegral = false }; - enum { IsArithmetic = false }; - enum { IsArray = true }; - }; -} - + enum { IsIntegral = false }; + enum { IsArithmetic = false }; + enum { IsArray = true }; + }; +} + #define UNIT_ASSERT_EQUAL_ENUM(expected, actual) UNIT_ASSERT_VALUES_EQUAL((bool)(expected), (bool)(actual)) - + Y_UNIT_TEST_SUITE(TTypeTraitsTestNg) { - template <typename T> - void TestImpl() { - //UNIT_ASSERT_EQUAL_ENUM(TTypeTraitsExpected<T>::IsPod, TTypeTraits<T>::IsPod); + template <typename T> + void TestImpl() { + //UNIT_ASSERT_EQUAL_ENUM(TTypeTraitsExpected<T>::IsPod, TTypeTraits<T>::IsPod); UNIT_ASSERT_EQUAL_ENUM(TTypeTraitsExpected<T>::IsVoid, std::is_void<T>::value); UNIT_ASSERT_EQUAL_ENUM(TTypeTraitsExpected<T>::IsEnum, std::is_enum<T>::value); UNIT_ASSERT_EQUAL_ENUM(TTypeTraitsExpected<T>::IsIntegral, std::is_integral<T>::value); @@ -397,30 +397,30 @@ Y_UNIT_TEST_SUITE(TTypeTraitsTestNg) { UNIT_ASSERT_EQUAL_ENUM(TTypeTraitsExpected<T>::IsRvalueReference, std::is_rvalue_reference<T>::value); UNIT_ASSERT_EQUAL_ENUM(TTypeTraitsExpected<T>::IsArray, std::is_array<T>::value); UNIT_ASSERT_EQUAL_ENUM(TTypeTraitsExpected<T>::IsClassType, std::is_class<T>::value); - } - + } + #define TYPE_TEST(name, type) \ Y_UNIT_TEST(name) { \ TestImpl<type>(); \ } - - TYPE_TEST(Void, void) - TYPE_TEST(Int, int) - TYPE_TEST(Float, float) - TYPE_TEST(LongDouble, long double) - TYPE_TEST(SizeT, size_t) - TYPE_TEST(VolatileInt, volatile int) - TYPE_TEST(ConstInt, const int) - TYPE_TEST(Enum, ETestEnum) - TYPE_TEST(FloatPointer, float*) - TYPE_TEST(FloatReference, float&) - TYPE_TEST(FloatConstReference, const float&) - TYPE_TEST(FloatArray, float[17]) + + TYPE_TEST(Void, void) + TYPE_TEST(Int, int) + TYPE_TEST(Float, float) + TYPE_TEST(LongDouble, long double) + TYPE_TEST(SizeT, size_t) + TYPE_TEST(VolatileInt, volatile int) + TYPE_TEST(ConstInt, const int) + TYPE_TEST(Enum, ETestEnum) + TYPE_TEST(FloatPointer, float*) + TYPE_TEST(FloatReference, float&) + TYPE_TEST(FloatConstReference, const float&) + TYPE_TEST(FloatArray, float[17]) TYPE_TEST(PodClass, TPodClass) TYPE_TEST(NonPodClass, TNonPodClass) TYPE_TEST(NonPodClassReference, TNonPodClass&) TYPE_TEST(NonPodClassConstReference, const TNonPodClass&) -} +} enum E4 { X diff --git a/util/generic/vector_ut.cpp b/util/generic/vector_ut.cpp index be27286038..0f6b4037a0 100644 --- a/util/generic/vector_ut.cpp +++ b/util/generic/vector_ut.cpp @@ -1,13 +1,13 @@ #include "vector.h" - + #include <library/cpp/testing/unittest/registar.h> #include <utility> #include "yexception.h" - + #include <stdexcept> -class TYVectorTest: public TTestBase { +class TYVectorTest: public TTestBase { UNIT_TEST_SUITE(TYVectorTest); UNIT_TEST(TestConstructorsAndAssignments) UNIT_TEST(TestTildeEmptyToNull) @@ -92,23 +92,23 @@ private: } // Copy-paste of STLPort tests - + void Test1() { TVector<int> v1; // Empty vector of integers. - + UNIT_ASSERT(v1.empty() == true); UNIT_ASSERT(v1.size() == 0); UNIT_ASSERT(!v1); - + // UNIT_ASSERT(v1.max_size() == INT_MAX / sizeof(int)); // cout << "max_size = " << v1.max_size() << endl; v1.push_back(42); // Add an integer to the vector. - + UNIT_ASSERT(v1.size() == 1); UNIT_ASSERT(v1); - + UNIT_ASSERT(v1[0] == 42); - + { TVector<TVector<int>> vect(10); TVector<TVector<int>>::iterator it(vect.begin()), end(vect.end()); @@ -117,102 +117,102 @@ private: UNIT_ASSERT((*it).size() == 0); UNIT_ASSERT((*it).capacity() == 0); UNIT_ASSERT((*it).begin() == (*it).end()); - } - } + } + } } - + void Test2() { TVector<double> v1; // Empty vector of doubles. v1.push_back(32.1); v1.push_back(40.5); TVector<double> v2; // Another empty vector of doubles. v2.push_back(3.56); - + UNIT_ASSERT(v1.size() == 2); UNIT_ASSERT(v1[0] == 32.1); UNIT_ASSERT(v1[1] == 40.5); - + UNIT_ASSERT(v2.size() == 1); UNIT_ASSERT(v2[0] == 3.56); v1.swap(v2); // Swap the vector's contents. - + UNIT_ASSERT(v1.size() == 1); UNIT_ASSERT(v1[0] == 3.56); - + UNIT_ASSERT(v2.size() == 2); UNIT_ASSERT(v2[0] == 32.1); UNIT_ASSERT(v2[1] == 40.5); - + v2 = v1; // Assign one vector to another. - + UNIT_ASSERT(v2.size() == 1); UNIT_ASSERT(v2[0] == 3.56); } - + void Test3() { using vec_type = TVector<char>; - + vec_type v1; // Empty vector of characters. v1.push_back('h'); v1.push_back('i'); - + UNIT_ASSERT(v1.size() == 2); UNIT_ASSERT(v1[0] == 'h'); UNIT_ASSERT(v1[1] == 'i'); - + vec_type v2(v1.begin(), v1.end()); v2[1] = 'o'; // Replace second character. - + UNIT_ASSERT(v2.size() == 2); UNIT_ASSERT(v2[0] == 'h'); UNIT_ASSERT(v2[1] == 'o'); - + UNIT_ASSERT((v1 == v2) == false); - + UNIT_ASSERT((v1 < v2) == true); } - + void Test4() { TVector<int> v(4); - + v[0] = 1; v[1] = 4; v[2] = 9; v[3] = 16; - + UNIT_ASSERT(v.front() == 1); UNIT_ASSERT(v.back() == 16); - + v.push_back(25); - + UNIT_ASSERT(v.back() == 25); UNIT_ASSERT(v.size() == 5); - + v.pop_back(); - + UNIT_ASSERT(v.back() == 16); UNIT_ASSERT(v.size() == 4); } - + void Test5() { int array[] = {1, 4, 9, 16}; - + TVector<int> v(array, array + 4); - + UNIT_ASSERT(v.size() == 4); - + UNIT_ASSERT(v[0] == 1); UNIT_ASSERT(v[1] == 4); UNIT_ASSERT(v[2] == 9); UNIT_ASSERT(v[3] == 16); } - + void Test6() { int array[] = {1, 4, 9, 16, 25, 36}; - + TVector<int> v(array, array + 6); TVector<int>::iterator vit; - + UNIT_ASSERT(v.size() == 6); UNIT_ASSERT(v[0] == 1); UNIT_ASSERT(v[1] == 4); @@ -220,57 +220,57 @@ private: UNIT_ASSERT(v[3] == 16); UNIT_ASSERT(v[4] == 25); UNIT_ASSERT(v[5] == 36); - + vit = v.erase(v.begin()); // Erase first element. UNIT_ASSERT(*vit == 4); - + UNIT_ASSERT(v.size() == 5); UNIT_ASSERT(v[0] == 4); UNIT_ASSERT(v[1] == 9); UNIT_ASSERT(v[2] == 16); UNIT_ASSERT(v[3] == 25); UNIT_ASSERT(v[4] == 36); - + vit = v.erase(v.end() - 1); // Erase last element. UNIT_ASSERT(vit == v.end()); - + UNIT_ASSERT(v.size() == 4); UNIT_ASSERT(v[0] == 4); UNIT_ASSERT(v[1] == 9); UNIT_ASSERT(v[2] == 16); UNIT_ASSERT(v[3] == 25); - + v.erase(v.begin() + 1, v.end() - 1); // Erase all but first and last. - + UNIT_ASSERT(v.size() == 2); UNIT_ASSERT(v[0] == 4); UNIT_ASSERT(v[1] == 25); } - + void Test7() { int array1[] = {1, 4, 25}; int array2[] = {9, 16}; - + TVector<int> v(array1, array1 + 3); TVector<int>::iterator vit; vit = v.insert(v.begin(), 0); // Insert before first element. UNIT_ASSERT(*vit == 0); - + vit = v.insert(v.end(), 36); // Insert after last element. UNIT_ASSERT(*vit == 36); - + UNIT_ASSERT(v.size() == 5); UNIT_ASSERT(v[0] == 0); UNIT_ASSERT(v[1] == 1); UNIT_ASSERT(v[2] == 4); UNIT_ASSERT(v[3] == 25); UNIT_ASSERT(v[4] == 36); - + // Insert contents of array2 before fourth element. v.insert(v.begin() + 3, array2, array2 + 2); - + UNIT_ASSERT(v.size() == 7); - + UNIT_ASSERT(v[0] == 0); UNIT_ASSERT(v[1] == 1); UNIT_ASSERT(v[2] == 4); @@ -278,11 +278,11 @@ private: UNIT_ASSERT(v[4] == 16); UNIT_ASSERT(v[5] == 25); UNIT_ASSERT(v[6] == 36); - + size_t curCapacity = v.capacity(); v.clear(); UNIT_ASSERT(v.empty()); - + //check that clear save reserved data UNIT_ASSERT_EQUAL(curCapacity, v.capacity()); @@ -294,22 +294,22 @@ private: UNIT_ASSERT(v[3] == 10); UNIT_ASSERT(v[4] == 10); } - + struct TestStruct { unsigned int a[3]; }; - + void TestCapacity() { { TVector<int> v; - + UNIT_ASSERT(v.capacity() == 0); v.push_back(42); UNIT_ASSERT(v.capacity() >= 1); v.reserve(5000); UNIT_ASSERT(v.capacity() >= 5000); } - + { TVector<int> v(Reserve(100)); @@ -322,18 +322,18 @@ private: TVector<TestStruct> va; va.reserve(1); va.reserve(2); - } + } } - + void TestAt() { TVector<int> v; TVector<int> const& cv = v; - + v.push_back(10); UNIT_ASSERT(v.at(0) == 10); v.at(0) = 20; UNIT_ASSERT(cv.at(0) == 20); - + for (;;) { try { v.at(1) = 20; @@ -342,15 +342,15 @@ private: return; } catch (...) { UNIT_ASSERT(false); - } - } + } + } } - + void TestPointer() { TVector<int*> v1; TVector<int*> v2 = v1; TVector<int*> v3; - + v3.insert(v3.end(), v1.begin(), v1.end()); } @@ -358,25 +358,25 @@ private: TVector<int> ref; for (int i = 0; i < 5; ++i) { ref.push_back(i); - } - + } + TVector<TVector<int>> v_v_int(1, ref); v_v_int.push_back(v_v_int[0]); v_v_int.push_back(ref); v_v_int.push_back(v_v_int[0]); v_v_int.push_back(v_v_int[0]); v_v_int.push_back(ref); - + TVector<TVector<int>>::iterator vvit(v_v_int.begin()), vvitEnd(v_v_int.end()); for (; vvit != vvitEnd; ++vvit) { UNIT_ASSERT(*vvit == ref); - } + } } - + struct Point { int x, y; }; - + struct PointEx: public Point { PointEx() : builtFromBase(false) @@ -386,37 +386,37 @@ private: : builtFromBase(true) { } - + bool builtFromBase; }; - + void TestIterators() { TVector<int> vint(10, 0); TVector<int> const& crvint = vint; - + UNIT_ASSERT(vint.begin() == vint.begin()); UNIT_ASSERT(crvint.begin() == vint.begin()); UNIT_ASSERT(vint.begin() == crvint.begin()); UNIT_ASSERT(crvint.begin() == crvint.begin()); - + UNIT_ASSERT(vint.begin() != vint.end()); UNIT_ASSERT(crvint.begin() != vint.end()); UNIT_ASSERT(vint.begin() != crvint.end()); UNIT_ASSERT(crvint.begin() != crvint.end()); - + UNIT_ASSERT(vint.rbegin() == vint.rbegin()); // Not Standard: //UNIT_ASSERT(vint.rbegin() == crvint.rbegin()); //UNIT_ASSERT(crvint.rbegin() == vint.rbegin()); UNIT_ASSERT(crvint.rbegin() == crvint.rbegin()); - + UNIT_ASSERT(vint.rbegin() != vint.rend()); // Not Standard: //UNIT_ASSERT(vint.rbegin() != crvint.rend()); //UNIT_ASSERT(crvint.rbegin() != vint.rend()); UNIT_ASSERT(crvint.rbegin() != crvint.rend()); } - + void TestShrink() { TVector<int> v; v.resize(1000); @@ -429,8 +429,8 @@ private: } /* This test check a potential issue with empty base class - * optimization. Some compilers (VC6) do not implement it - * correctly resulting ina wrong behavior. */ + * optimization. Some compilers (VC6) do not implement it + * correctly resulting ina wrong behavior. */ void TestEbo() { // We use heap memory as test failure can corrupt vector internal // representation making executable crash on vector destructor invocation. @@ -439,36 +439,36 @@ private: using V = TVector<int>; V* pv1 = new V(1, 1); V* pv2 = new V(10, 2); - + size_t v1Capacity = pv1->capacity(); size_t v2Capacity = pv2->capacity(); - + pv1->swap(*pv2); - + UNIT_ASSERT(pv1->size() == 10); UNIT_ASSERT(pv1->capacity() == v2Capacity); UNIT_ASSERT((*pv1)[5] == 2); - + UNIT_ASSERT(pv2->size() == 1); UNIT_ASSERT(pv2->capacity() == v1Capacity); UNIT_ASSERT((*pv2)[0] == 1); - + delete pv2; delete pv1; } - + void TestFillInConstructor() { for (int k = 0; k < 3; ++k) { TVector<int> v(100); UNIT_ASSERT_VALUES_EQUAL(100u, v.size()); for (size_t i = 0; i < v.size(); ++i) { UNIT_ASSERT_VALUES_EQUAL(0, v[i]); - } + } // fill with garbage for the next iteration for (size_t i = 0; i < v.size(); ++i) { v[i] = 10; } - } + } } struct TPod { @@ -591,6 +591,6 @@ private: CheckInitializeList(v); } } -}; - -UNIT_TEST_SUITE_REGISTRATION(TYVectorTest); +}; + +UNIT_TEST_SUITE_REGISTRATION(TYVectorTest); diff --git a/util/generic/ylimits.h b/util/generic/ylimits.h index e6a52a340f..fe42b4dfc0 100644 --- a/util/generic/ylimits.h +++ b/util/generic/ylimits.h @@ -2,10 +2,10 @@ #include <limits> -#if defined(max) || defined(min) +#if defined(max) || defined(min) #error "stop defining 'min' and 'max' macros, evil people" -#endif - +#endif + template <class T> static constexpr T Max() noexcept { return std::numeric_limits<T>::max(); diff --git a/util/generic/ymath_ut.cpp b/util/generic/ymath_ut.cpp index 18103fa3c9..29190b55eb 100644 --- a/util/generic/ymath_ut.cpp +++ b/util/generic/ymath_ut.cpp @@ -12,13 +12,13 @@ template <class T> static inline T SlowClp2(T t) noexcept { Y_ASSERT(t > 0); - T ret = 1; + T ret = 1; - while (ret < t) { - ret *= 2; + while (ret < t) { + ret *= 2; } - return ret; + return ret; } class TMathTest: public TTestBase { @@ -50,7 +50,7 @@ private: inline void TestIsValidFloat() { UNIT_ASSERT(IsValidFloat(-Max<double>() / 2.)); } - + inline void TestClpSimple() { UNIT_ASSERT_EQUAL(FastClp2<ui32>(12), 16); UNIT_ASSERT_EQUAL(FastClp2<ui16>(11), 16); @@ -83,19 +83,19 @@ void TMathTest::TestSqr() { } void TMathTest::TestClp2() { - for (ui8 i = 1; i < 127; ++i) { + for (ui8 i = 1; i < 127; ++i) { UNIT_ASSERT_EQUAL(SlowClp2(i), FastClp2(i)); } - for (ui16 i = 1; i < 255; ++i) { + for (ui16 i = 1; i < 255; ++i) { UNIT_ASSERT_EQUAL(SlowClp2(i), FastClp2(i)); } - for (ui32 i = 1; i < 255; ++i) { + for (ui32 i = 1; i < 255; ++i) { UNIT_ASSERT_EQUAL(SlowClp2(i), FastClp2(i)); } - for (ui64 i = 1; i < 255; ++i) { + for (ui64 i = 1; i < 255; ++i) { UNIT_ASSERT_EQUAL(SlowClp2(i), FastClp2(i)); } diff --git a/util/memory/blob.cpp b/util/memory/blob.cpp index 9610cf82ac..91da5cadca 100644 --- a/util/memory/blob.cpp +++ b/util/memory/blob.cpp @@ -159,7 +159,7 @@ TBlob TBlob::SubBlob(size_t len) const { TBlob TBlob::SubBlob(size_t begin, size_t end) const { if (begin > Length() || end > Length() || begin > end) { - ythrow yexception() << "incorrect subblob (" << begin << ", " << end << ", outer length = " << Length() << ")"; + ythrow yexception() << "incorrect subblob (" << begin << ", " << end << ", outer length = " << Length() << ")"; } return TBlob(Begin() + begin, end - begin, S_.Base); diff --git a/util/network/address.cpp b/util/network/address.cpp index 5e13cdca81..a81a9e6994 100644 --- a/util/network/address.cpp +++ b/util/network/address.cpp @@ -37,8 +37,8 @@ static inline void PrintAddr(IOutputStream& out, const IRemoteAddr& addr) { if (printPort) { out << "[" << buf << "]" << ":" << InetToHost(sa->sin6_port); - } else { - out << buf; + } else { + out << buf; } break; @@ -55,33 +55,33 @@ static inline void PrintAddr(IOutputStream& out, const IRemoteAddr& addr) { #endif default: { - size_t len = addr.Len(); - + size_t len = addr.Len(); + const char* b = (const char*)a; - const char* e = b + len; - - bool allZeros = true; - for (size_t i = 0; i < len; ++i) { - if (b[i] != 0) { - allZeros = false; - break; - } - } - - if (allZeros) { - out << "(raw all zeros)"; - } else { + const char* e = b + len; + + bool allZeros = true; + for (size_t i = 0; i < len; ++i) { + if (b[i] != 0) { + allZeros = false; + break; + } + } + + if (allZeros) { + out << "(raw all zeros)"; + } else { out << "(raw " << (int)a->sa_family << " "; - - while (b != e) { - //just print raw bytes - out << (int)*b++; - if (b != e) { - out << " "; - } - } - - out << ")"; + + while (b != e) { + //just print raw bytes + out << (int)*b++; + if (b != e) { + out << " "; + } + } + + out << ")"; } break; diff --git a/util/network/address_ut.cpp b/util/network/address_ut.cpp index 2a18cad9df..28f45172ff 100644 --- a/util/network/address_ut.cpp +++ b/util/network/address_ut.cpp @@ -1,33 +1,33 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "address.h" - -using namespace NAddr; - + +#include "address.h" + +using namespace NAddr; + Y_UNIT_TEST_SUITE(IRemoteAddr_ToString) { Y_UNIT_TEST(Raw) { - THolder<TOpaqueAddr> opaque(new TOpaqueAddr); - IRemoteAddr* addr = opaque.Get(); - + THolder<TOpaqueAddr> opaque(new TOpaqueAddr); + IRemoteAddr* addr = opaque.Get(); + TString s = ToString(*addr); - UNIT_ASSERT_VALUES_EQUAL("(raw all zeros)", s); - - opaque->MutableAddr()->sa_data[10] = 17; - + UNIT_ASSERT_VALUES_EQUAL("(raw all zeros)", s); + + opaque->MutableAddr()->sa_data[10] = 17; + TString t = ToString(*addr); - + UNIT_ASSERT_C(t.StartsWith("(raw 0 0"), t); UNIT_ASSERT_C(t.EndsWith(')'), t); - } - + } + Y_UNIT_TEST(Ipv6) { - TNetworkAddress address("::1", 22); - TNetworkAddress::TIterator it = address.Begin(); - UNIT_ASSERT(it != address.End()); - UNIT_ASSERT(it->ai_family == AF_INET6); + TNetworkAddress address("::1", 22); + TNetworkAddress::TIterator it = address.Begin(); + UNIT_ASSERT(it != address.End()); + UNIT_ASSERT(it->ai_family == AF_INET6); TString toString = ToString((const IRemoteAddr&)TAddrInfo(&*it)); UNIT_ASSERT_VALUES_EQUAL(TString("[::1]:22"), toString); - } + } Y_UNIT_TEST(Loopback) { TNetworkAddress localAddress("127.70.0.1", 22); @@ -36,4 +36,4 @@ Y_UNIT_TEST_SUITE(IRemoteAddr_ToString) { TNetworkAddress localAddress2("127.0.0.1", 22); UNIT_ASSERT_VALUES_EQUAL(NAddr::IsLoopback(TAddrInfo(&*localAddress2.Begin())), true); } -} +} diff --git a/util/network/ip.h b/util/network/ip.h index 0e944248a5..dc7c2d24a0 100644 --- a/util/network/ip.h +++ b/util/network/ip.h @@ -8,10 +8,10 @@ #include <util/generic/string.h> #include <util/generic/yexception.h> -/// IPv4 address in network format +/// IPv4 address in network format using TIpHost = ui32; - -/// Port number in host format + +/// Port number in host format using TIpPort = ui16; /* @@ -53,7 +53,7 @@ static inline TIpHost ResolveHost(const char* data, size_t len) { return HostToInet(ret); } -/// socket address +/// socket address struct TIpAddress: public sockaddr_in { inline TIpAddress() noexcept { Clear(); diff --git a/util/network/poller.cpp b/util/network/poller.cpp index 36a007b5c6..7954d0e8b5 100644 --- a/util/network/poller.cpp +++ b/util/network/poller.cpp @@ -57,18 +57,18 @@ void TSocketPoller::WaitRdhup(SOCKET sock, void* cookie) { Impl_->Set(cookie, sock, CONT_POLL_RDHUP); } -void TSocketPoller::WaitReadOneShot(SOCKET sock, void* cookie) { - Impl_->Set(cookie, sock, CONT_POLL_READ | CONT_POLL_ONE_SHOT); -} - -void TSocketPoller::WaitWriteOneShot(SOCKET sock, void* cookie) { - Impl_->Set(cookie, sock, CONT_POLL_WRITE | CONT_POLL_ONE_SHOT); -} - -void TSocketPoller::WaitReadWriteOneShot(SOCKET sock, void* cookie) { - Impl_->Set(cookie, sock, CONT_POLL_READ | CONT_POLL_WRITE | CONT_POLL_ONE_SHOT); -} - +void TSocketPoller::WaitReadOneShot(SOCKET sock, void* cookie) { + Impl_->Set(cookie, sock, CONT_POLL_READ | CONT_POLL_ONE_SHOT); +} + +void TSocketPoller::WaitWriteOneShot(SOCKET sock, void* cookie) { + Impl_->Set(cookie, sock, CONT_POLL_WRITE | CONT_POLL_ONE_SHOT); +} + +void TSocketPoller::WaitReadWriteOneShot(SOCKET sock, void* cookie) { + Impl_->Set(cookie, sock, CONT_POLL_READ | CONT_POLL_WRITE | CONT_POLL_ONE_SHOT); +} + void TSocketPoller::WaitReadWriteEdgeTriggered(SOCKET sock, void* cookie) { Impl_->Set(cookie, sock, CONT_POLL_READ | CONT_POLL_WRITE | CONT_POLL_EDGE_TRIGGERED); } diff --git a/util/network/poller.h b/util/network/poller.h index d687fb0463..8dccd73140 100644 --- a/util/network/poller.h +++ b/util/network/poller.h @@ -18,7 +18,7 @@ public: void WaitReadOneShot(SOCKET sock, void* cookie); void WaitWriteOneShot(SOCKET sock, void* cookie); void WaitReadWriteOneShot(SOCKET sock, void* cookie); - + void WaitReadWriteEdgeTriggered(SOCKET sock, void* cookie); void RestartReadWriteEdgeTriggered(SOCKET sock, void* cookie, bool empty = true); diff --git a/util/network/poller_ut.cpp b/util/network/poller_ut.cpp index 1d542f8c67..6df0dda8ec 100644 --- a/util/network/poller_ut.cpp +++ b/util/network/poller_ut.cpp @@ -1,105 +1,105 @@ #include <library/cpp/testing/unittest/registar.h> #include <util/system/error.h> - + #include "pair.h" -#include "poller.h" +#include "poller.h" #include "pollerimpl.h" - + Y_UNIT_TEST_SUITE(TSocketPollerTest) { Y_UNIT_TEST(TestSimple) { - SOCKET sockets[2]; - UNIT_ASSERT(SocketPair(sockets) == 0); - - TSocketHolder s1(sockets[0]); - TSocketHolder s2(sockets[1]); - - TSocketPoller poller; + SOCKET sockets[2]; + UNIT_ASSERT(SocketPair(sockets) == 0); + + TSocketHolder s1(sockets[0]); + TSocketHolder s2(sockets[1]); + + TSocketPoller poller; poller.WaitRead(sockets[1], (void*)17); - + UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); - - for (ui32 i = 0; i < 3; ++i) { + + for (ui32 i = 0; i < 3; ++i) { char buf[] = {18}; - UNIT_ASSERT_VALUES_EQUAL(1, send(sockets[0], buf, 1, 0)); - + UNIT_ASSERT_VALUES_EQUAL(1, send(sockets[0], buf, 1, 0)); + UNIT_ASSERT_VALUES_EQUAL((void*)17, poller.WaitT(TDuration::Zero())); - - UNIT_ASSERT_VALUES_EQUAL(1, recv(sockets[1], buf, 1, 0)); - UNIT_ASSERT_VALUES_EQUAL(18, buf[0]); - + + UNIT_ASSERT_VALUES_EQUAL(1, recv(sockets[1], buf, 1, 0)); + UNIT_ASSERT_VALUES_EQUAL(18, buf[0]); + UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); - } - } - + } + } + Y_UNIT_TEST(TestSimpleOneShot) { - SOCKET sockets[2]; - UNIT_ASSERT(SocketPair(sockets) == 0); - - TSocketHolder s1(sockets[0]); - TSocketHolder s2(sockets[1]); - - TSocketPoller poller; - + SOCKET sockets[2]; + UNIT_ASSERT(SocketPair(sockets) == 0); + + TSocketHolder s1(sockets[0]); + TSocketHolder s2(sockets[1]); + + TSocketPoller poller; + UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); - - for (ui32 i = 0; i < 3; ++i) { + + for (ui32 i = 0; i < 3; ++i) { poller.WaitReadOneShot(sockets[1], (void*)17); - - char buf[1]; - - buf[0] = i + 20; - - UNIT_ASSERT_VALUES_EQUAL(1, send(sockets[0], buf, 1, 0)); - + + char buf[1]; + + buf[0] = i + 20; + + UNIT_ASSERT_VALUES_EQUAL(1, send(sockets[0], buf, 1, 0)); + UNIT_ASSERT_VALUES_EQUAL((void*)17, poller.WaitT(TDuration::Zero())); - - UNIT_ASSERT_VALUES_EQUAL(1, recv(sockets[1], buf, 1, 0)); - UNIT_ASSERT_VALUES_EQUAL(char(i + 20), buf[0]); - - UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); - UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); - - buf[0] = i + 21; - - UNIT_ASSERT_VALUES_EQUAL(1, send(sockets[0], buf, 1, 0)); - - // this fails if socket is not oneshot - UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); - UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); - - UNIT_ASSERT_VALUES_EQUAL(1, recv(sockets[1], buf, 1, 0)); - UNIT_ASSERT_VALUES_EQUAL(char(i + 21), buf[0]); - } - } - + + UNIT_ASSERT_VALUES_EQUAL(1, recv(sockets[1], buf, 1, 0)); + UNIT_ASSERT_VALUES_EQUAL(char(i + 20), buf[0]); + + UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); + UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); + + buf[0] = i + 21; + + UNIT_ASSERT_VALUES_EQUAL(1, send(sockets[0], buf, 1, 0)); + + // this fails if socket is not oneshot + UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); + UNIT_ASSERT_VALUES_EQUAL(nullptr, poller.WaitT(TDuration::Zero())); + + UNIT_ASSERT_VALUES_EQUAL(1, recv(sockets[1], buf, 1, 0)); + UNIT_ASSERT_VALUES_EQUAL(char(i + 21), buf[0]); + } + } + Y_UNIT_TEST(TestItIsSafeToUnregisterUnregisteredDescriptor) { - SOCKET sockets[2]; - UNIT_ASSERT(SocketPair(sockets) == 0); - - TSocketHolder s1(sockets[0]); - TSocketHolder s2(sockets[1]); - - TSocketPoller poller; - - poller.Unwait(s1); - } - + SOCKET sockets[2]; + UNIT_ASSERT(SocketPair(sockets) == 0); + + TSocketHolder s1(sockets[0]); + TSocketHolder s2(sockets[1]); + + TSocketPoller poller; + + poller.Unwait(s1); + } + Y_UNIT_TEST(TestItIsSafeToReregisterDescriptor) { - SOCKET sockets[2]; - UNIT_ASSERT(SocketPair(sockets) == 0); - - TSocketHolder s1(sockets[0]); - TSocketHolder s2(sockets[1]); - - TSocketPoller poller; - + SOCKET sockets[2]; + UNIT_ASSERT(SocketPair(sockets) == 0); + + TSocketHolder s1(sockets[0]); + TSocketHolder s2(sockets[1]); + + TSocketPoller poller; + poller.WaitRead(s1, nullptr); poller.WaitRead(s1, nullptr); poller.WaitWrite(s1, nullptr); - } + } Y_UNIT_TEST(TestSimpleEdgeTriggered) { SOCKET sockets[2]; @@ -233,4 +233,4 @@ Y_UNIT_TEST_SUITE(TSocketPollerTest) { UNIT_ASSERT_EQUAL(TPoller::ExtractEvent(&e), (void*)17); } #endif -} +} diff --git a/util/network/pollerimpl.h b/util/network/pollerimpl.h index 35a981ea7d..e8c7e40fba 100644 --- a/util/network/pollerimpl.h +++ b/util/network/pollerimpl.h @@ -9,7 +9,7 @@ #include <util/generic/utility.h> #include <util/generic/vector.h> #include <util/generic/yexception.h> -#include <util/datetime/base.h> +#include <util/datetime/base.h> #if defined(_freebsd_) || defined(_darwin_) #define HAVE_KQUEUE_POLLER @@ -426,7 +426,7 @@ public: inline void SetImpl(void* data, SOCKET fd, int what) { with_lock (CommandLock_) { - Commands_.push_back(TCommand(fd, what, data)); + Commands_.push_back(TCommand(fd, what, data)); } Signal(); @@ -434,7 +434,7 @@ public: inline void Remove(SOCKET fd) noexcept { with_lock (CommandLock_) { - Commands_.push_back(TCommand(fd, 0)); + Commands_.push_back(TCommand(fd, 0)); } Signal(); @@ -442,7 +442,7 @@ public: inline size_t Wait(TEvent* events, size_t len, int timeout) noexcept { auto guard = Guard(Lock_); - + do { if (Begin_ != End_) { const size_t ret = Min<size_t>(End_ - Begin_, len); @@ -475,25 +475,25 @@ public: inline size_t WaitBase(TEvent* events, size_t len, int timeout) noexcept { with_lock (CommandLock_) { for (auto command = Commands_.begin(); command != Commands_.end(); ++command) { - if (command->Filter_ != 0) { - Fds_.Set(command->Fd_, command->Cookie_, command->Filter_); - } else { - Fds_.Remove(command->Fd_); - } - } - - Commands_.clear(); - } - - TTempBuf tmpBuf(3 * sizeof(fd_set) + Fds_.size() * sizeof(SOCKET)); + if (command->Filter_ != 0) { + Fds_.Set(command->Fd_, command->Cookie_, command->Filter_); + } else { + Fds_.Remove(command->Fd_); + } + } + + Commands_.clear(); + } + + TTempBuf tmpBuf(3 * sizeof(fd_set) + Fds_.size() * sizeof(SOCKET)); fd_set* in = (fd_set*)tmpBuf.Data(); fd_set* out = &in[1]; fd_set* errFds = &in[2]; SOCKET* keysToDeleteBegin = (SOCKET*)&in[3]; - SOCKET* keysToDeleteEnd = keysToDeleteBegin; - + SOCKET* keysToDeleteEnd = keysToDeleteBegin; + #if defined(_msan_enabled_) // msan doesn't handle FD_ZERO and cause false positive BALANCER-1347 memset(in, 0, sizeof(*in)); memset(out, 0, sizeof(*out)); @@ -529,12 +529,12 @@ public: if (FD_ISSET(fd, errFds)) { (events++)->Error(handle.Data(), EIO); - - if (handle.Filter() & CONT_POLL_ONE_SHOT) { - *keysToDeleteEnd = fd; - ++keysToDeleteEnd; - } - + + if (handle.Filter() & CONT_POLL_ONE_SHOT) { + *keysToDeleteEnd = fd; + ++keysToDeleteEnd; + } + } else { int what = 0; @@ -548,11 +548,11 @@ public: if (what) { (events++)->Success(handle.Data(), what); - - if (handle.Filter() & CONT_POLL_ONE_SHOT) { - *keysToDeleteEnd = fd; - ++keysToDeleteEnd; - } + + if (handle.Filter() & CONT_POLL_ONE_SHOT) { + *keysToDeleteEnd = fd; + ++keysToDeleteEnd; + } if (handle.Filter() & CONT_POLL_EDGE_TRIGGERED) { // Emulate edge-triggered for level-triggered select(). @@ -563,11 +563,11 @@ public: } } - while (keysToDeleteBegin != keysToDeleteEnd) { - Fds_.erase(*keysToDeleteBegin); - ++keysToDeleteBegin; - } - + while (keysToDeleteBegin != keysToDeleteEnd) { + Fds_.erase(*keysToDeleteBegin); + ++keysToDeleteBegin; + } + return events - eventsStart; } @@ -611,25 +611,25 @@ private: } private: - struct TCommand { - SOCKET Fd_; - int Filter_; // 0 to remove - void* Cookie_; - - TCommand(SOCKET fd, int filter, void* cookie) + struct TCommand { + SOCKET Fd_; + int Filter_; // 0 to remove + void* Cookie_; + + TCommand(SOCKET fd, int filter, void* cookie) : Fd_(fd) , Filter_(filter) , Cookie_(cookie) { } - - TCommand(SOCKET fd, int filter) + + TCommand(SOCKET fd, int filter) : Fd_(fd) , Filter_(filter) { } - }; - + }; + TFds Fds_; TMyMutex Lock_; @@ -637,9 +637,9 @@ private: TEvent* Begin_; TEvent* End_; - TMyMutex CommandLock_; + TMyMutex CommandLock_; TVector<TCommand> Commands_; - + SOCKET Signal_[2]; }; #endif diff --git a/util/network/socket.cpp b/util/network/socket.cpp index f3ea77929a..4f6e804346 100644 --- a/util/network/socket.cpp +++ b/util/network/socket.cpp @@ -554,25 +554,25 @@ static ssize_t DoSendMsg(SOCKET sock, const struct iovec* iov, int iovcnt) { #endif void TSocketHolder::Close() noexcept { - if (Fd_ != INVALID_SOCKET) { - bool ok = (closesocket(Fd_) == 0); - if (!ok) { + if (Fd_ != INVALID_SOCKET) { + bool ok = (closesocket(Fd_) == 0); + if (!ok) { // Do not quietly close bad descriptor, // because often it means double close // that is disasterous -#ifdef _win_ +#ifdef _win_ Y_VERIFY(WSAGetLastError() != WSAENOTSOCK, "must not quietly close bad socket descriptor"); -#elif defined(_unix_) +#elif defined(_unix_) Y_VERIFY(errno != EBADF, "must not quietly close bad descriptor: fd=%d", int(Fd_)); -#else +#else #error unsupported platform -#endif - } - - Fd_ = INVALID_SOCKET; - } -} - +#endif + } + + Fd_ = INVALID_SOCKET; + } +} + class TSocket::TImpl: public TAtomicRefCount<TImpl> { using TOps = TSocket::TOps; diff --git a/util/random/random.cpp b/util/random/random.cpp index a2191076b8..71f9323856 100644 --- a/util/random/random.cpp +++ b/util/random/random.cpp @@ -79,7 +79,7 @@ namespace { return GetRndGen<TToRealType<TY>::TResult>()->Uniform(n); \ } -DEF_RND(char) +DEF_RND(char) DEF_RND(unsigned char) DEF_RND(unsigned int) DEF_RND(unsigned long) @@ -89,11 +89,11 @@ DEF_RND(unsigned long long) #undef DEF_RND template <> -bool RandomNumber<bool>() { - return RandomNumber<ui8>() % 2 == 0; -} - -template <> +bool RandomNumber<bool>() { + return RandomNumber<ui8>() % 2 == 0; +} + +template <> float RandomNumber<float>() { float ret; diff --git a/util/random/random.h b/util/random/random.h index 8e7e23bf87..16b52d3995 100644 --- a/util/random/random.h +++ b/util/random/random.h @@ -5,7 +5,7 @@ * * specialized for: * all unsigned types (return value in range [0, MAX_VALUE_FOR_TYPE]) - * bool + * bool * long double (return value in range [0, 1)) * double (return value in range [0, 1)) * float (return value in range [0, 1)) @@ -16,8 +16,8 @@ T RandomNumber(); /* * returns value in range [0, max) */ -template <class T> -T RandomNumber(T max); +template <class T> +T RandomNumber(T max); /* * Re-initialize random state - useful after forking in multi-process programs. diff --git a/util/random/random_ut.cpp b/util/random/random_ut.cpp index c2ffc6ab27..30427676f3 100644 --- a/util/random/random_ut.cpp +++ b/util/random/random_ut.cpp @@ -1,9 +1,9 @@ #include "random.h" #include <library/cpp/testing/unittest/registar.h> - -#include <util/generic/ylimits.h> - + +#include <util/generic/ylimits.h> + template <class T> static inline void AssertRange(T v, T r1, T r2) { UNIT_ASSERT(v >= r1); @@ -11,50 +11,50 @@ static inline void AssertRange(T v, T r1, T r2) { } Y_UNIT_TEST_SUITE(TRandomNumberTest) { - template <typename T> - void TestAll(T n) { - for (T i = 0; i < n; ++i) { - while (RandomNumber<T>(n) != i) { - } - } - } - - template <typename T> - void TestSome(T n) { - for (int i = 0; i < 100; ++i) { - UNIT_ASSERT(RandomNumber<T>(n) < n); - } - } - - template <typename T> - void TestType() { - TestAll<T>(1); - TestAll<T>(2); - TestAll<T>(3); - TestAll<T>(4); - TestAll<T>(5); - TestAll<T>(6); - TestAll<T>(9); - TestAll<T>(15); - TestAll<T>(16); - TestSome<T>(Max<T>()); - TestSome<T>(Max<T>() - 1); - TestSome<T>(Max<T>() - 2); - TestSome<T>(Max<T>() - 3); - TestSome<T>(Max<T>() - 4); - TestSome<T>(Max<T>() - 5); - TestSome<T>(Max<T>() - 7); - TestSome<T>(Max<T>() - 8); - TestSome<T>(Max<T>() - 2222); - TestSome<T>(Max<T>() - 22222); - } - + template <typename T> + void TestAll(T n) { + for (T i = 0; i < n; ++i) { + while (RandomNumber<T>(n) != i) { + } + } + } + + template <typename T> + void TestSome(T n) { + for (int i = 0; i < 100; ++i) { + UNIT_ASSERT(RandomNumber<T>(n) < n); + } + } + + template <typename T> + void TestType() { + TestAll<T>(1); + TestAll<T>(2); + TestAll<T>(3); + TestAll<T>(4); + TestAll<T>(5); + TestAll<T>(6); + TestAll<T>(9); + TestAll<T>(15); + TestAll<T>(16); + TestSome<T>(Max<T>()); + TestSome<T>(Max<T>() - 1); + TestSome<T>(Max<T>() - 2); + TestSome<T>(Max<T>() - 3); + TestSome<T>(Max<T>() - 4); + TestSome<T>(Max<T>() - 5); + TestSome<T>(Max<T>() - 7); + TestSome<T>(Max<T>() - 8); + TestSome<T>(Max<T>() - 2222); + TestSome<T>(Max<T>() - 22222); + } + Y_UNIT_TEST(TestWithLimit) { - TestType<unsigned short>(); - TestType<unsigned int>(); - TestType<unsigned long>(); - TestType<unsigned long long>(); - } + TestType<unsigned short>(); + TestType<unsigned int>(); + TestType<unsigned long>(); + TestType<unsigned long long>(); + } Y_UNIT_TEST(TestRandomNumberFloat) { for (size_t i = 0; i < 1000; ++i) { @@ -73,13 +73,13 @@ Y_UNIT_TEST_SUITE(TRandomNumberTest) { AssertRange<long double>(RandomNumber<long double>(), 0.0, 1.0); } } - + Y_UNIT_TEST(TestBoolean) { - while (RandomNumber<bool>()) { - } - while (!RandomNumber<bool>()) { - } - } + while (RandomNumber<bool>()) { + } + while (!RandomNumber<bool>()) { + } + } Y_UNIT_TEST(TestResetSeed) { SetRandomSeed(42); @@ -152,4 +152,4 @@ Y_UNIT_TEST_SUITE(TRandomNumberTest) { UNIT_ASSERT_EQUAL(RandomNumber<ui32>(1 << 8), el); } } -} +} diff --git a/util/stream/format.h b/util/stream/format.h index 30e321d377..b033208a1b 100644 --- a/util/stream/format.h +++ b/util/stream/format.h @@ -1,5 +1,5 @@ #pragma once - + #include "mem.h" #include "output.h" @@ -8,7 +8,7 @@ #include <util/generic/flags.h> #include <util/memory/tempbuf.h> #include <util/string/cast.h> - + enum ENumberFormatFlag { HF_FULL = 0x01, /**< Output number with leading zeros. */ HF_ADDX = 0x02, /**< Output '0x' or '0b' before hex/bin digits. */ @@ -21,7 +21,7 @@ enum ESizeFormat { SF_BYTES, /**< Base 1024, byte suffix. 1100 gets turned into "1.07KiB". */ }; -namespace NFormatPrivate { +namespace NFormatPrivate { template <size_t Value> struct TLog2: std::integral_constant<size_t, TLog2<Value / 2>::value + 1> {}; @@ -29,66 +29,66 @@ namespace NFormatPrivate { struct TLog2<1>: std::integral_constant<size_t, 0> {}; static inline void WriteChars(IOutputStream& os, char c, size_t count) { - if (count == 0) - return; - TTempBuf buf(count); - memset(buf.Data(), c, count); - os.Write(buf.Data(), count); - } - - template <typename T> - struct TLeftPad { + if (count == 0) + return; + TTempBuf buf(count); + memset(buf.Data(), c, count); + os.Write(buf.Data(), count); + } + + template <typename T> + struct TLeftPad { T Value; - size_t Width; - char Padc; - + size_t Width; + char Padc; + inline TLeftPad(const T& value, size_t width, char padc) - : Value(value) - , Width(width) - , Padc(padc) - { - } - }; - - template <typename T> + : Value(value) + , Width(width) + , Padc(padc) + { + } + }; + + template <typename T> IOutputStream& operator<<(IOutputStream& o, const TLeftPad<T>& lp) { - TTempBuf buf; - TMemoryOutput ss(buf.Data(), buf.Size()); + TTempBuf buf; + TMemoryOutput ss(buf.Data(), buf.Size()); ss << lp.Value; - size_t written = buf.Size() - ss.Avail(); - if (lp.Width > written) { - WriteChars(o, lp.Padc, lp.Width - written); - } - o.Write(buf.Data(), written); - return o; - } - - template <typename T> - struct TRightPad { + size_t written = buf.Size() - ss.Avail(); + if (lp.Width > written) { + WriteChars(o, lp.Padc, lp.Width - written); + } + o.Write(buf.Data(), written); + return o; + } + + template <typename T> + struct TRightPad { T Value; - size_t Width; - char Padc; - + size_t Width; + char Padc; + inline TRightPad(const T& value, size_t width, char padc) - : Value(value) - , Width(width) - , Padc(padc) - { - } - }; - - template <typename T> + : Value(value) + , Width(width) + , Padc(padc) + { + } + }; + + template <typename T> IOutputStream& operator<<(IOutputStream& o, const TRightPad<T>& lp) { - TTempBuf buf; - TMemoryOutput ss(buf.Data(), buf.Size()); + TTempBuf buf; + TMemoryOutput ss(buf.Data(), buf.Size()); ss << lp.Value; - size_t written = buf.Size() - ss.Avail(); - o.Write(buf.Data(), written); - if (lp.Width > written) { - WriteChars(o, lp.Padc, lp.Width - written); - } - return o; - } + size_t written = buf.Size() - ss.Avail(); + o.Write(buf.Data(), written); + if (lp.Width > written) { + WriteChars(o, lp.Padc, lp.Width - written); + } + return o; + } template <typename T, size_t Base> struct TBaseNumber { @@ -184,8 +184,8 @@ namespace NFormatPrivate { double Value; ESizeFormat Format; }; -} - +} + /** * Output manipulator basically equivalent to `std::setw` and `std::setfill` * combined. @@ -203,11 +203,11 @@ namespace NFormatPrivate { * @param padc Character to use for padding. * @see RightPad */ -template <typename T> +template <typename T> static constexpr ::NFormatPrivate::TLeftPad<T> LeftPad(const T& value, const size_t width, const char padc = ' ') noexcept { return ::NFormatPrivate::TLeftPad<T>(value, width, padc); -} - +} + template <typename T, int N> static constexpr ::NFormatPrivate::TLeftPad<const T*> LeftPad(const T (&value)[N], const size_t width, const char padc = ' ') noexcept { return ::NFormatPrivate::TLeftPad<const T*>(value, width, padc); @@ -229,11 +229,11 @@ static constexpr ::NFormatPrivate::TLeftPad<const T*> LeftPad(const T (&value)[N * @param padc Character to use for padding. * @see LeftPad */ -template <typename T> +template <typename T> static constexpr ::NFormatPrivate::TRightPad<T> RightPad(const T& value, const size_t width, const char padc = ' ') noexcept { return ::NFormatPrivate::TRightPad<T>(value, width, padc); -} - +} + template <typename T, int N> static constexpr ::NFormatPrivate::TRightPad<const T*> RightPad(const T (&value)[N], const size_t width, const char padc = ' ') noexcept { return ::NFormatPrivate::TRightPad<const T*>(value, width, padc); diff --git a/util/stream/format_ut.cpp b/util/stream/format_ut.cpp index 2e50f0f9c5..43245aeb48 100644 --- a/util/stream/format_ut.cpp +++ b/util/stream/format_ut.cpp @@ -2,28 +2,28 @@ #include <library/cpp/testing/unittest/registar.h> #include <util/charset/wide.h> - + Y_UNIT_TEST_SUITE(TOutputStreamFormattingTest) { Y_UNIT_TEST(TestLeftPad) { - TStringStream ss; - ss << LeftPad(10, 4, '0'); - UNIT_ASSERT_VALUES_EQUAL("0010", ss.Str()); - + TStringStream ss; + ss << LeftPad(10, 4, '0'); + UNIT_ASSERT_VALUES_EQUAL("0010", ss.Str()); + ss.Clear(); - ss << LeftPad(222, 1); - UNIT_ASSERT_VALUES_EQUAL("222", ss.Str()); - } - + ss << LeftPad(222, 1); + UNIT_ASSERT_VALUES_EQUAL("222", ss.Str()); + } + Y_UNIT_TEST(TestRightPad) { - TStringStream ss; - ss << RightPad("aa", 4); - UNIT_ASSERT_VALUES_EQUAL("aa ", ss.Str()); - + TStringStream ss; + ss << RightPad("aa", 4); + UNIT_ASSERT_VALUES_EQUAL("aa ", ss.Str()); + ss.Clear(); - ss << RightPad("aa", 1); - UNIT_ASSERT_VALUES_EQUAL("aa", ss.Str()); - } - + ss << RightPad("aa", 1); + UNIT_ASSERT_VALUES_EQUAL("aa", ss.Str()); + } + Y_UNIT_TEST(TestTime) { TStringStream ss; @@ -179,4 +179,4 @@ Y_UNIT_TEST_SUITE(TOutputStreamFormattingTest) { UNIT_ASSERT_VALUES_EQUAL(ToString(HumanReadable(TDuration::Seconds(3672))), "1h 1m 12s"); UNIT_ASSERT_VALUES_EQUAL(ToString(HumanReadable(TDuration::Seconds(4220))), "1h 10m 20s"); } -} +} diff --git a/util/stream/output.cpp b/util/stream/output.cpp index c193889e7f..db81b81b70 100644 --- a/util/stream/output.cpp +++ b/util/stream/output.cpp @@ -99,7 +99,7 @@ void Out<std::string>(IOutputStream& o, const std::string& p) { o.Write(p.data(), p.length()); } -template <> +template <> void Out<std::string_view>(IOutputStream& o, const std::string_view& p) { o.Write(p.data(), p.length()); } diff --git a/util/stream/output.h b/util/stream/output.h index 8d904b5ecf..00eef50b95 100644 --- a/util/stream/output.h +++ b/util/stream/output.h @@ -296,7 +296,7 @@ static inline void Flush(IOutputStream& o) { /* * Also see format.h for additional manipulators. */ - + #include "debug.h" void RedirectStdioToAndroidLog(bool redirect); diff --git a/util/string/escape.cpp b/util/string/escape.cpp index afec4780e7..cd09a7dbd0 100644 --- a/util/string/escape.cpp +++ b/util/string/escape.cpp @@ -1,10 +1,10 @@ #include "escape.h" #include "cast.h" -#include <util/system/defaults.h> +#include <util/system/defaults.h> #include <util/charset/utf8.h> #include <util/charset/wide.h> - + /// @todo: escape trigraphs (eg "??/" is "\") /* REFEREBCES FOR ESCAPE SEQUENCE INTERPRETATION: @@ -54,31 +54,31 @@ namespace { template <typename TChar> static inline char OctDigit(TChar value) { Y_ASSERT(value < 8); - return '0' + value; + return '0' + value; } - + template <typename TChar> static inline bool IsPrintable(TChar c) { return c >= 32 && c <= 126; } - + template <typename TChar> static inline bool IsHexDigit(TChar c) { return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); } - + template <typename TChar> static inline bool IsOctDigit(TChar c) { return c >= '0' && c <= '7'; } - + template <typename TChar> struct TEscapeUtil; - + template <> struct TEscapeUtil<char> { static const size_t ESCAPE_C_BUFFER_SIZE = 4; - + template <typename TNextChar, typename TBufferChar> static inline size_t EscapeC(unsigned char c, TNextChar next, TBufferChar r[ESCAPE_C_BUFFER_SIZE]) { // (1) Printable characters go as-is, except backslash and double quote. @@ -124,13 +124,13 @@ namespace { r[3] = OctDigit((c & 0007) >> 0); return 4; } - } + } }; - + template <> struct TEscapeUtil<wchar16> { static const size_t ESCAPE_C_BUFFER_SIZE = 6; - + template <typename TNextChar, typename TBufferChar> static inline size_t EscapeC(wchar16 c, TNextChar next, TBufferChar r[ESCAPE_C_BUFFER_SIZE]) { if (c < 0x100) { @@ -144,26 +144,26 @@ namespace { r[5] = HexDigit((c & 0x000F) >> 0); return 6; } - } + } }; } - -template <class TChar> + +template <class TChar> TBasicString<TChar>& EscapeCImpl(const TChar* str, size_t len, TBasicString<TChar>& r) { using TEscapeUtil = ::TEscapeUtil<TChar>; - + TChar buffer[TEscapeUtil::ESCAPE_C_BUFFER_SIZE]; - + size_t i, j; for (i = 0, j = 0; i < len; ++i) { size_t rlen = TEscapeUtil::EscapeC(str[i], (i + 1 < len ? str[i + 1] : 0), buffer); - + if (rlen > 1) { r.append(str + j, i - j); j = i + 1; r.append(buffer, rlen); } - } + } if (j > 0) { r.append(str + j, len - j); @@ -172,8 +172,8 @@ TBasicString<TChar>& EscapeCImpl(const TChar* str, size_t len, TBasicString<TCha } return r; -} - +} + template TString& EscapeCImpl<TString::TChar>(const TString::TChar* str, size_t len, TString& r); template TUtf16String& EscapeCImpl<TUtf16String::TChar>(const TUtf16String::TChar* str, size_t len, TUtf16String& r); @@ -410,11 +410,11 @@ TUtf16String& EscapeC(const TWtringBuf str, TUtf16String& w) { TString EscapeC(const TString& str) { return EscapeC(str.data(), str.size()); -} - +} + TUtf16String EscapeC(const TUtf16String& str) { return EscapeC(str.data(), str.size()); -} +} TString& UnescapeC(const TStringBuf str, TString& s) { return UnescapeC(str.data(), str.size(), s); diff --git a/util/string/escape.h b/util/string/escape.h index ec72f71573..b01be65b0e 100644 --- a/util/string/escape.h +++ b/util/string/escape.h @@ -1,18 +1,18 @@ #pragma once - + #include <util/generic/string.h> #include <util/generic/strbuf.h> - -template <class TChar> + +template <class TChar> TBasicString<TChar>& EscapeCImpl(const TChar* str, size_t len, TBasicString<TChar>&); - + template <class TChar> TBasicString<TChar>& UnescapeCImpl(const TChar* str, size_t len, TBasicString<TChar>&); template <class TChar> TChar* UnescapeC(const TChar* str, size_t len, TChar* buf); -template <typename TChar> +template <typename TChar> static inline TBasicString<TChar>& EscapeC(const TChar* str, size_t len, TBasicString<TChar>& s) { return EscapeCImpl(str, len, s); } @@ -23,7 +23,7 @@ static inline TBasicString<TChar> EscapeC(const TChar* str, size_t len) { return EscapeC(str, len, s); } -template <typename TChar> +template <typename TChar> static inline TBasicString<TChar> EscapeC(const TBasicStringBuf<TChar>& str) { return EscapeC(str.data(), str.size()); } diff --git a/util/string/escape_ut.cpp b/util/string/escape_ut.cpp index 51836a64ed..cd38ecffd3 100644 --- a/util/string/escape_ut.cpp +++ b/util/string/escape_ut.cpp @@ -1,10 +1,10 @@ #include "escape.h" #include <library/cpp/testing/unittest/registar.h> - + #include <util/generic/string.h> -#include <util/charset/wide.h> - +#include <util/charset/wide.h> + using namespace std::string_view_literals; namespace { @@ -24,7 +24,7 @@ static const TExample CommonTestData[] = { // Should be valid UTF-8. {"http://ya.ru/", "http://ya.ru/"}, {"http://ya.ru/\\x17\\n", "http://ya.ru/\x17\n"}, - + {"http://ya.ru/\\0", "http://ya.ru/\0"sv}, {"http://ya.ru/\\0\\0", "http://ya.ru/\0\0"sv}, {"http://ya.ru/\\0\\0000", "http://ya.ru/\0\0" @@ -60,9 +60,9 @@ Y_UNIT_TEST_SUITE(TEscapeCTest) { UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\\x17\\n\\xAB", EscapeC(TString("http://ya.ru/\x17\n\xab"))); UNIT_ASSERT_VALUES_EQUAL("http://ya.ru/\x17\n\xab", UnescapeC(TString("http://ya.ru/\\x17\\n\\xAB"))); - UNIT_ASSERT_VALUES_EQUAL("h", EscapeC('h')); + UNIT_ASSERT_VALUES_EQUAL("h", EscapeC('h')); UNIT_ASSERT_VALUES_EQUAL("h", UnescapeC(TString("h"))); - UNIT_ASSERT_VALUES_EQUAL("\\xFF", EscapeC('\xFF')); + UNIT_ASSERT_VALUES_EQUAL("\\xFF", EscapeC('\xFF')); UNIT_ASSERT_VALUES_EQUAL("\xFF", UnescapeC(TString("\\xFF"))); UNIT_ASSERT_VALUES_EQUAL("\\377f", EscapeC(TString("\xff" @@ -77,8 +77,8 @@ Y_UNIT_TEST_SUITE(TEscapeCTest) { UnescapeC(TString("\\xFFg"))); UNIT_ASSERT_VALUES_EQUAL("\xEA\x9A\x96", UnescapeC(TString("\\uA696"))); UNIT_ASSERT_VALUES_EQUAL("Странный компроматтест", UnescapeC(TString("\\u0421\\u0442\\u0440\\u0430\\u043d\\u043d\\u044b\\u0439 \\u043a\\u043e\\u043c\\u043f\\u0440\\u043e\\u043c\\u0430\\u0442тест"))); - } - + } + Y_UNIT_TEST(TestWtrokaEscapeC) { for (const auto& e : CommonTestData) { TUtf16String expected(UTF8ToWide(e.Expected)); @@ -93,7 +93,7 @@ Y_UNIT_TEST_SUITE(TEscapeCTest) { UNIT_ASSERT_VALUES_EQUAL(u"http://ya.ru/\\x17\\n\\u1234", EscapeC(u"http://ya.ru/\x17\n\u1234")); UNIT_ASSERT_VALUES_EQUAL(u"h", EscapeC(u'h')); UNIT_ASSERT_VALUES_EQUAL(u"\\xFF", EscapeC(wchar16(255))); - } + } Y_UNIT_TEST(TestEscapeTrigraphs) { UNIT_ASSERT_VALUES_EQUAL("?", EscapeC(TString("?"))); @@ -145,4 +145,4 @@ Y_UNIT_TEST_SUITE(TEscapeCTest) { UNIT_ASSERT_VALUES_EQUAL(UnescapeC("\\U00000020"), " "); UNIT_ASSERT_VALUES_EQUAL(UnescapeC("\\Uxxx"), "Uxxx"); } -} +} diff --git a/util/string/split.cpp b/util/string/split.cpp index 7438c07525..7d26857cc7 100644 --- a/util/string/split.cpp +++ b/util/string/split.cpp @@ -1,24 +1,24 @@ -#include "split.h" - +#include "split.h" + template <class TValue> inline size_t Split(const char* ptr, const char* delim, TVector<TValue>& values) { - values.erase(values.begin(), values.end()); - while (ptr && *ptr) { - ptr += strspn(ptr, delim); - if (ptr && *ptr) { - size_t epos = strcspn(ptr, delim); - assert(epos); + values.erase(values.begin(), values.end()); + while (ptr && *ptr) { + ptr += strspn(ptr, delim); + if (ptr && *ptr) { + size_t epos = strcspn(ptr, delim); + assert(epos); values.push_back(TValue(ptr, epos)); - ptr += epos; - } - } - return values.size(); -} - + ptr += epos; + } + } + return values.size(); +} + size_t Split(const char* ptr, const char* delim, TVector<TString>& values) { return Split<TString>(ptr, delim, values); } size_t Split(const TString& in, const TString& delim, TVector<TString>& res) { return Split(in.data(), delim.data(), res); -} +} diff --git a/util/string/split.h b/util/string/split.h index 166ee10660..bc46d9e64c 100644 --- a/util/string/split.h +++ b/util/string/split.h @@ -407,10 +407,10 @@ static inline void Split(char* buf, char ch, T* res) { SplitString(buf, delim, pusher); } -/// Split string into res vector. Res vector is cleared before split. +/// Split string into res vector. Res vector is cleared before split. /// Old good slow split function. /// Field delimter is any number of symbols specified in delim (no empty strings in res vector) -/// @return number of elements created +/// @return number of elements created size_t Split(const char* in, const char* delim, TVector<TString>& res); size_t Split(const TString& in, const TString& delim, TVector<TString>& res); diff --git a/util/system/backtrace.cpp b/util/system/backtrace.cpp index 1bc57de75d..b77fe58fb1 100644 --- a/util/system/backtrace.cpp +++ b/util/system/backtrace.cpp @@ -236,15 +236,15 @@ TResolvedSymbol ResolveSymbol(void* sym, char*, size_t) { void FormatBackTrace(IOutputStream* out, void* const* backtrace, size_t backtraceSize) { char tmpBuf[1024]; - for (size_t i = 0; i < backtraceSize; ++i) { - TResolvedSymbol rs = ResolveSymbol(backtrace[i], tmpBuf, sizeof(tmpBuf)); + for (size_t i = 0; i < backtraceSize; ++i) { + TResolvedSymbol rs = ResolveSymbol(backtrace[i], tmpBuf, sizeof(tmpBuf)); *out << rs.Name << "+" << ((ptrdiff_t)backtrace[i] - (ptrdiff_t)rs.NearestSymbol) << " (" << Hex((ptrdiff_t)backtrace[i], HF_ADDX) << ')' << '\n'; } } TFormatBackTraceFn FormatBackTraceFn = FormatBackTrace; - + TFormatBackTraceFn SetFormatBackTraceFn(TFormatBackTraceFn f) { TFormatBackTraceFn prevFn = FormatBackTraceFn; FormatBackTraceFn = f; @@ -264,25 +264,25 @@ TFormatBackTraceFn GetFormatBackTraceFn() { void PrintBackTrace() { FormatBackTrace(&Cerr); } - -TBackTrace::TBackTrace() - : Size(0) + +TBackTrace::TBackTrace() + : Size(0) { } - -void TBackTrace::Capture() { - Size = BackTrace(Data, CAPACITY); -} - + +void TBackTrace::Capture() { + Size = BackTrace(Data, CAPACITY); +} + void TBackTrace::PrintTo(IOutputStream& out) const { FormatBackTraceFn(&out, Data, Size); -} - +} + TString TBackTrace::PrintToString() const { - TStringStream ss; - PrintTo(ss); + TStringStream ss; + PrintTo(ss); return ss.Str(); -} +} size_t TBackTrace::size() const { return Size; diff --git a/util/system/backtrace.h b/util/system/backtrace.h index 71b3c8f528..2fce7585c3 100644 --- a/util/system/backtrace.h +++ b/util/system/backtrace.h @@ -17,7 +17,7 @@ TResolvedSymbol ResolveSymbol(void* sym, char* buf, size_t len); void FormatBackTrace(IOutputStream* out, void* const* backtrace, size_t backtraceSize); void FormatBackTrace(IOutputStream* out); void PrintBackTrace(); - + using TFormatBackTraceFn = void (*)(IOutputStream*, void* const* backtrace, size_t backtraceSize); TFormatBackTraceFn SetFormatBackTraceFn(TFormatBackTraceFn f); @@ -25,15 +25,15 @@ TFormatBackTraceFn GetFormatBackTraceFn(); using TBackTraceView = TArrayRef<void* const>; -class TBackTrace { -private: +class TBackTrace { +private: static constexpr size_t CAPACITY = 300; - void* Data[CAPACITY]; - size_t Size; + void* Data[CAPACITY]; + size_t Size; -public: - TBackTrace(); - void Capture(); +public: + TBackTrace(); + void Capture(); void PrintTo(IOutputStream&) const; TString PrintToString() const; size_t size() const; @@ -41,4 +41,4 @@ public: operator TBackTraceView() const; static TBackTrace FromCurrentException(); -}; +}; diff --git a/util/system/byteorder_ut.cpp b/util/system/byteorder_ut.cpp index b728f86915..39b8603d3f 100644 --- a/util/system/byteorder_ut.cpp +++ b/util/system/byteorder_ut.cpp @@ -1,8 +1,8 @@ #include "byteorder.h" #include <library/cpp/testing/unittest/registar.h> - -class TByteOrderTest: public TTestBase { + +class TByteOrderTest: public TTestBase { UNIT_TEST_SUITE(TByteOrderTest); UNIT_TEST(TestSwap16) UNIT_TEST(TestSwap32) @@ -21,6 +21,6 @@ private: inline void TestSwap64() { UNIT_ASSERT_EQUAL(0x1234567890abcdefULL, SwapBytes((ui64)ULL(0xefcdab9078563412))); } -}; - -UNIT_TEST_SUITE_REGISTRATION(TByteOrderTest); +}; + +UNIT_TEST_SUITE_REGISTRATION(TByteOrderTest); diff --git a/util/system/compat.cpp b/util/system/compat.cpp index edeb78e8e3..18fbfa296a 100644 --- a/util/system/compat.cpp +++ b/util/system/compat.cpp @@ -16,7 +16,7 @@ #include <io.h> #endif -#ifndef HAVE_NATIVE_GETPROGNAME +#ifndef HAVE_NATIVE_GETPROGNAME const char* getprogname() { return GetProgramName().data(); } diff --git a/util/system/compat.h b/util/system/compat.h index 6aef2761f9..c53dbcca17 100644 --- a/util/system/compat.h +++ b/util/system/compat.h @@ -30,9 +30,9 @@ extern "C" { #if defined(__FreeBSD__) || defined(_darwin_) #define HAVE_NATIVE_GETPROGNAME -#endif - -#ifndef HAVE_NATIVE_GETPROGNAME +#endif + +#ifndef HAVE_NATIVE_GETPROGNAME const char* getprogname(); #endif diff --git a/util/system/compat_ut.cpp b/util/system/compat_ut.cpp index a46b41a4ce..dbd9289c17 100644 --- a/util/system/compat_ut.cpp +++ b/util/system/compat_ut.cpp @@ -1,12 +1,12 @@ #include "compat.h" #include <library/cpp/testing/unittest/registar.h> - -#include <util/folder/dirut.h> -#include <util/stream/output.h> - + +#include <util/folder/dirut.h> +#include <util/stream/output.h> + Y_UNIT_TEST_SUITE(TCompatTest) { Y_UNIT_TEST(TestGetprogname) { - getprogname(); // just check it links - } -} + getprogname(); // just check it links + } +} diff --git a/util/system/condvar.cpp b/util/system/condvar.cpp index 48addfb210..62f3d22356 100644 --- a/util/system/condvar.cpp +++ b/util/system/condvar.cpp @@ -112,7 +112,7 @@ public: int ret = pthread_cond_timedwait(&Cond_, (pthread_mutex_t*)lock.Handle(), &spec); Y_VERIFY(ret == 0 || ret == ETIMEDOUT, "pthread_cond_timedwait failed: %s", LastSystemErrorText(ret)); - + return ret == 0; } } diff --git a/util/system/condvar.h b/util/system/condvar.h index 2fa33a1107..569162717c 100644 --- a/util/system/condvar.h +++ b/util/system/condvar.h @@ -4,7 +4,7 @@ #include <util/generic/ptr.h> #include <util/generic/noncopyable.h> -#include <util/datetime/base.h> +#include <util/datetime/base.h> #include <utility> diff --git a/util/system/context.cpp b/util/system/context.cpp index 0c8f3069ff..ad99309088 100644 --- a/util/system/context.cpp +++ b/util/system/context.cpp @@ -24,7 +24,7 @@ namespace __cxxabiv1 { #endif #endif -#include <util/stream/output.h> +#include <util/stream/output.h> #include <util/generic/yexception.h> #define FROM_CONTEXT_IMPL @@ -37,7 +37,7 @@ void ITrampoLine::DoRunNaked() { try { DoRun(); } catch (...) { - Cerr << "Uncaught exception in coroutine: " << CurrentExceptionMessage() << "\n"; + Cerr << "Uncaught exception in coroutine: " << CurrentExceptionMessage() << "\n"; } abort(); diff --git a/util/system/datetime.cpp b/util/system/datetime.cpp index 4c5488a24c..b07b50679a 100644 --- a/util/system/datetime.cpp +++ b/util/system/datetime.cpp @@ -7,7 +7,7 @@ #include <ctime> #include <cerrno> - + #ifdef _darwin_ #include <AvailabilityMacros.h> #if defined(MAC_OS_X_VERSION_10_12) && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 @@ -19,8 +19,8 @@ static ui64 ToMicroSeconds(const struct timeval& tv) { return (ui64)tv.tv_sec * 1000000 + (ui64)tv.tv_usec; -} - +} + #if defined(_win_) static ui64 ToMicroSeconds(const FILETIME& ft) { return (((ui64)ft.dwHighDateTime << 32) + (ui64)ft.dwLowDateTime) / (ui64)10; @@ -32,12 +32,12 @@ static ui64 ToMicroSeconds(const struct timespec& ts) { #endif ui64 MicroSeconds() noexcept { - struct timeval tv; + struct timeval tv; gettimeofday(&tv, nullptr); - + return ToMicroSeconds(tv); -} - +} + ui64 ThreadCPUUserTime() noexcept { #if defined(_win_) FILETIME creationTime, exitTime, kernelTime, userTime; @@ -73,26 +73,26 @@ ui64 ThreadCPUTime() noexcept { } ui32 Seconds() noexcept { - struct timeval tv; + struct timeval tv; gettimeofday(&tv, nullptr); - return tv.tv_sec; -} - + return tv.tv_sec; +} + void NanoSleep(ui64 ns) noexcept { #if defined(_win_) - Sleep(ns / 1000000); -#else - const ui64 NS = 1000 * 1000 * 1000; - struct timespec req; - req.tv_sec = ns / NS; - req.tv_nsec = ns % NS; - struct timespec left; - while (nanosleep(&req, &left) < 0) { + Sleep(ns / 1000000); +#else + const ui64 NS = 1000 * 1000 * 1000; + struct timespec req; + req.tv_sec = ns / NS; + req.tv_nsec = ns % NS; + struct timespec left; + while (nanosleep(&req, &left) < 0) { Y_ASSERT(errno == EINTR); - req = left; - } -#endif -} + req = left; + } +#endif +} #if defined(_x86_) extern const bool HaveRdtscp = NX86::HaveRDTSCP(); diff --git a/util/system/datetime.h b/util/system/datetime.h index 92aacdc31f..aa009974e0 100644 --- a/util/system/datetime.h +++ b/util/system/datetime.h @@ -1,21 +1,21 @@ #pragma once - + #include "defaults.h" #include "platform.h" - + #if defined(_win_) #include <intrin.h> #pragma intrinsic(__rdtsc) #endif // _win_ - + #if defined(_darwin_) && !defined(_x86_) #include <mach/mach_time.h> #endif -/// util/system/datetime.h contains only system time providers -/// for handy datetime utilities include util/datetime/base.h - -/// Current time in microseconds since epoch +/// util/system/datetime.h contains only system time providers +/// for handy datetime utilities include util/datetime/base.h + +/// Current time in microseconds since epoch ui64 MicroSeconds() noexcept; /// Current time in milliseconds since epoch inline ui64 MilliSeconds() { @@ -25,17 +25,17 @@ inline ui64 MilliSeconds() { inline ui64 millisec() { return MilliSeconds(); } -/// Current time in seconds since epoch +/// Current time in seconds since epoch ui32 Seconds() noexcept; ///Current thread time in microseconds ui64 ThreadCPUUserTime() noexcept; ui64 ThreadCPUSystemTime() noexcept; ui64 ThreadCPUTime() noexcept; - + void NanoSleep(ui64 ns) noexcept; - -// GetCycleCount guarantees to return synchronous values on different cores -// and provide constant rate only on modern Intel and AMD processors + +// GetCycleCount guarantees to return synchronous values on different cores +// and provide constant rate only on modern Intel and AMD processors // NOTE: rdtscp is used to prevent out of order execution // rdtsc can be reordered, while rdtscp cannot be reordered // with preceding instructions @@ -95,4 +95,4 @@ Y_FORCE_INLINE ui64 GetCycleCount() noexcept { #else #error "unsupported arch" #endif -} +} diff --git a/util/system/datetime_ut.cpp b/util/system/datetime_ut.cpp index bddf9df35d..a865a888ca 100644 --- a/util/system/datetime_ut.cpp +++ b/util/system/datetime_ut.cpp @@ -1 +1 @@ -#include "datetime.h" +#include "datetime.h" diff --git a/util/system/defaults.h b/util/system/defaults.h index 2d95db149b..dcd7abea38 100644 --- a/util/system/defaults.h +++ b/util/system/defaults.h @@ -142,7 +142,7 @@ constexpr bool Y_IS_DEBUG_BUILD = true; #define Y_CAT_II(X, Y) X##Y #define Y_STRINGIZE(X) UTIL_PRIVATE_STRINGIZE_AUX(X) -#define UTIL_PRIVATE_STRINGIZE_AUX(X) #X +#define UTIL_PRIVATE_STRINGIZE_AUX(X) #X #if defined(__COUNTER__) #define Y_GENERATE_UNIQUE_ID(N) Y_CAT(N, __COUNTER__) diff --git a/util/system/error_ut.cpp b/util/system/error_ut.cpp index 8403dfa180..763b0dddb7 100644 --- a/util/system/error_ut.cpp +++ b/util/system/error_ut.cpp @@ -6,7 +6,7 @@ #ifdef _win_ #include "winint.h" -#else +#else #include <fcntl.h> #endif diff --git a/util/system/event.h b/util/system/event.h index a63498a72e..cab2fc478a 100644 --- a/util/system/event.h +++ b/util/system/event.h @@ -1,7 +1,7 @@ #pragma once #include <util/generic/ptr.h> -#include <util/datetime/base.h> +#include <util/datetime/base.h> struct TEventResetType { enum ResetMode { diff --git a/util/system/execpath.cpp b/util/system/execpath.cpp index 836edeb76d..33198af58b 100644 --- a/util/system/execpath.cpp +++ b/util/system/execpath.cpp @@ -2,11 +2,11 @@ #include <stdlib.h> -#if defined(_solaris_) +#if defined(_solaris_) #include <stdlib.h> -#elif defined(_darwin_) +#elif defined(_darwin_) #include <mach-o/dyld.h> -#elif defined(_win_) +#elif defined(_win_) #include "winint.h" #include <io.h> #elif defined(_linux_) @@ -16,13 +16,13 @@ #include <sys/types.h> // for u_int not defined in sysctl.h #include <sys/sysctl.h> #include <unistd.h> -#endif - +#endif + #include <util/folder/dirut.h> #include <util/generic/singleton.h> #include <util/generic/function.h> #include <util/generic/yexception.h> -#include <util/memory/tempbuf.h> +#include <util/memory/tempbuf.h> #include <util/stream/file.h> #include <util/stream/pipe.h> #include <util/string/cast.h> @@ -30,7 +30,7 @@ #include "filemap.h" #include "execpath.h" #include "fs.h" - + #if defined(_freebsd_) static inline bool GoodPath(const TString& path) { return path.find('/') != TString::npos; @@ -101,9 +101,9 @@ static inline bool FreeBSDGuessExecBasePath(const TString& guessBasePath, TStrin #endif static TString GetExecPathImpl() { -#if defined(_solaris_) +#if defined(_solaris_) return execname(); -#elif defined(_darwin_) +#elif defined(_darwin_) TTempBuf execNameBuf; for (size_t i = 0; i < 2; ++i) { std::remove_pointer_t<TFunctionArg<decltype(_NSGetExecutablePath), 1>> bufsize = execNameBuf.Size(); @@ -112,10 +112,10 @@ static TString GetExecPathImpl() { return execNameBuf.Data(); } else if (r == -1) { execNameBuf = TTempBuf(bufsize); - } + } } ythrow yexception() << "GetExecPathImpl() failed"; -#elif defined(_win_) +#elif defined(_win_) TTempBuf execNameBuf; for (;;) { DWORD r = GetModuleFileName(nullptr, execNameBuf.Data(), execNameBuf.Size()); @@ -125,13 +125,13 @@ static TString GetExecPathImpl() { ythrow yexception() << "GetExecPathImpl() failed: " << LastSystemErrorText(); } else { return execNameBuf.Data(); - } + } } #elif defined(_linux_) || defined(_cygwin_) TString path("/proc/self/exe"); return NFs::ReadLink(path); // TODO(yoda): check if the filename ends with " (deleted)" -#elif defined(_freebsd_) +#elif defined(_freebsd_) TString execPath = FreeBSDGetExecPath(); if (GoodPath(execPath)) { return execPath; @@ -150,9 +150,9 @@ static TString GetExecPathImpl() { } ythrow yexception() << "can not resolve exec path"; -#else +#else #error dont know how to implement GetExecPath on this platform -#endif +#endif } static bool GetPersistentExecPathImpl(TString& to) { @@ -189,10 +189,10 @@ namespace { TString PersistentExecPath; }; } - + const TString& GetExecPath() { return TExecPathsHolder::Instance()->ExecPath; -} +} const TString& GetPersistentExecPath() { return TExecPathsHolder::Instance()->PersistentExecPath; diff --git a/util/system/execpath.h b/util/system/execpath.h index 7f1cd5a14b..4b914b8e85 100644 --- a/util/system/execpath.h +++ b/util/system/execpath.h @@ -1,7 +1,7 @@ #pragma once - + #include <util/generic/fwd.h> - + // NOTE: This function has rare sporadic failures (throws exceptions) on FreeBSD. See REVIEW:54297 const TString& GetExecPath(); diff --git a/util/system/execpath_ut.cpp b/util/system/execpath_ut.cpp index ebccec6791..16b01466f5 100644 --- a/util/system/execpath_ut.cpp +++ b/util/system/execpath_ut.cpp @@ -3,8 +3,8 @@ #include <library/cpp/testing/unittest/registar.h> #include "platform.h" -#include <util/folder/dirut.h> - +#include <util/folder/dirut.h> + Y_UNIT_TEST_SUITE(TExecPathTest) { Y_UNIT_TEST(TestIt) { TString execPath = GetExecPath(); @@ -18,5 +18,5 @@ Y_UNIT_TEST_SUITE(TExecPathTest) { throw; } - } -} + } +} diff --git a/util/system/fasttime.cpp b/util/system/fasttime.cpp index 3a32bd4019..057a814f0a 100644 --- a/util/system/fasttime.cpp +++ b/util/system/fasttime.cpp @@ -5,7 +5,7 @@ #include <util/generic/yexception.h> #include <utility> -#include <util/thread/singleton.h> +#include <util/thread/singleton.h> #if defined(_win_) || defined(_arm32_) || defined(_cygwin_) ui64 InterpolatedMicroSeconds() { @@ -236,7 +236,7 @@ namespace { } ui64 InterpolatedMicroSeconds() { - return FastTlsSingleton<TTimePredictor>()->Get(); + return FastTlsSingleton<TTimePredictor>()->Get(); } #endif diff --git a/util/system/file.cpp b/util/system/file.cpp index 5fd4ae8d2f..4a261d020c 100644 --- a/util/system/file.cpp +++ b/util/system/file.cpp @@ -22,7 +22,7 @@ #include <util/generic/yexception.h> #include <util/datetime/base.h> - + #include <errno.h> #if defined(_unix_) @@ -140,7 +140,7 @@ TFileHandle::TFileHandle(const TString& fName, EOpenMode oMode) noexcept { ::SetFilePointer(Fd_, 0, 0, FILE_END); } -#elif defined(_unix_) +#elif defined(_unix_) switch (createMode) { case OpenExisting: @@ -262,7 +262,7 @@ TFileHandle::TFileHandle(const TString& fName, EOpenMode oMode) noexcept { if (Fd_ >= 0 && (oMode & Transient)) { unlink(fName.data()); } -#else +#else #error unsupported platform #endif } @@ -277,9 +277,9 @@ bool TFileHandle::Close() noexcept { Y_VERIFY(GetLastError() != ERROR_INVALID_HANDLE, "must not quietly close invalid handle"); } -#elif defined(_unix_) +#elif defined(_unix_) if (Fd_ != INVALID_FHANDLE) { - isOk = (::close(Fd_) == 0 || errno == EINTR); + isOk = (::close(Fd_) == 0 || errno == EINTR); } if (!isOk) { // Do not quietly close bad descriptor, @@ -307,7 +307,7 @@ static inline i64 DoSeek(FHANDLE h, i64 offset, SeekDir origin) noexcept { pos.QuadPart = -1; } return pos.QuadPart; -#elif defined(_unix_) +#elif defined(_unix_) static int dir[] = {SEEK_SET, SEEK_CUR, SEEK_END}; #if defined(_sun_) return ::llseek(h, (offset_t)offset, dir[origin]); @@ -328,9 +328,9 @@ i64 TFileHandle::Seek(i64 offset, SeekDir origin) noexcept { } i64 TFileHandle::GetLength() const noexcept { - // XXX: returns error code, but does not set errno + // XXX: returns error code, but does not set errno if (!IsOpen()) { - return -1L; + return -1L; } return GetFileLength(Fd_); } @@ -424,7 +424,7 @@ bool TFileHandle::Flush() noexcept { * The function returns FALSE, and GetLastError returns ERROR_INVALID_HANDLE. */ return ok || GetLastError() == ERROR_INVALID_HANDLE; -#elif defined(_unix_) +#elif defined(_unix_) int ret = ::fsync(Fd_); /* @@ -469,13 +469,13 @@ i32 TFileHandle::Read(void* buffer, ui32 byteCount) noexcept { return bytesRead; } return -1; -#elif defined(_unix_) +#elif defined(_unix_) i32 ret; do { ret = ::read(Fd_, buffer, byteCount); } while (ret == -1 && errno == EINTR); return ret; -#else +#else #error unsupported platform #endif } @@ -490,13 +490,13 @@ i32 TFileHandle::Write(const void* buffer, ui32 byteCount) noexcept { return bytesWritten; } return -1; -#elif defined(_unix_) +#elif defined(_unix_) i32 ret; do { ret = ::write(Fd_, buffer, byteCount); } while (ret == -1 && errno == EINTR); return ret; -#else +#else #error unsupported platform #endif } @@ -515,13 +515,13 @@ i32 TFileHandle::Pread(void* buffer, ui32 byteCount, i64 offset) const noexcept return 0; } return -1; -#elif defined(_unix_) +#elif defined(_unix_) i32 ret; do { ret = ::pread(Fd_, buffer, byteCount, offset); } while (ret == -1 && errno == EINTR); return ret; -#else +#else #error unsupported platform #endif } @@ -537,13 +537,13 @@ i32 TFileHandle::Pwrite(const void* buffer, ui32 byteCount, i64 offset) const no return bytesWritten; } return -1; -#elif defined(_unix_) +#elif defined(_unix_) i32 ret; do { ret = ::pwrite(Fd_, buffer, byteCount, offset); } while (ret == -1 && errno == EINTR); return ret; -#else +#else #error unsupported platform #endif } @@ -558,9 +558,9 @@ FHANDLE TFileHandle::Duplicate() const noexcept { return INVALID_FHANDLE; } return dupHandle; -#elif defined(_unix_) +#elif defined(_unix_) return ::dup(Fd_); -#else +#else #error unsupported platform #endif } @@ -776,10 +776,10 @@ bool TFileHandle::FlushCache(i64 offset, i64 length, bool wait) noexcept { } TString DecodeOpenMode(ui32 mode0) { - ui32 mode = mode0; - + ui32 mode = mode0; + TStringBuilder r; - + #define F(flag) \ if ((mode & flag) == flag) { \ mode &= ~flag; \ @@ -787,14 +787,14 @@ TString DecodeOpenMode(ui32 mode0) { r << TStringBuf("|"); \ } \ r << TStringBuf(#flag); \ - } - - F(RdWr) + } + + F(RdWr) F(RdOnly) F(WrOnly) F(CreateAlways) - F(CreateNew) + F(CreateNew) F(OpenAlways) F(TruncExisting) F(ForAppend) @@ -809,38 +809,38 @@ TString DecodeOpenMode(ui32 mode0) { F(NoReuse) F(NoReadAhead) - F(AX) - F(AR) - F(AW) - F(ARW) - - F(AXOther) - F(AWOther) - F(AROther) - F(AXGroup) - F(AWGroup) - F(ARGroup) - F(AXUser) - F(AWUser) - F(ARUser) - -#undef F - - if (mode != 0) { + F(AX) + F(AR) + F(AW) + F(ARW) + + F(AXOther) + F(AWOther) + F(AROther) + F(AXGroup) + F(AWGroup) + F(ARGroup) + F(AXUser) + F(AWUser) + F(ARUser) + +#undef F + + if (mode != 0) { if (r) { r << TStringBuf("|"); } r << Hex(mode); - } - + } + if (!r) { return "0"; } - - return r; + + return r; } - + class TFile::TImpl: public TAtomicRefCount<TImpl> { public: inline TImpl(FHANDLE fd, const TString& fname = TString()) @@ -1277,7 +1277,7 @@ TFile Duplicate(int fd) { return TFile(dupHandle); #elif defined(_unix_) - return TFile(::dup(fd)); + return TFile(::dup(fd)); #else #error unsupported platform #endif diff --git a/util/system/file.h b/util/system/file.h index 27f711aff6..9502e159b6 100644 --- a/util/system/file.h +++ b/util/system/file.h @@ -53,7 +53,7 @@ Y_DECLARE_FLAGS(EOpenMode, EOpenModeFlag) Y_DECLARE_OPERATORS_FOR_FLAGS(EOpenMode) TString DecodeOpenMode(ui32 openMode); - + enum SeekDir { sSet = 0, sCur = 1, @@ -64,7 +64,7 @@ class TFileHandle: public TNonCopyable { public: constexpr TFileHandle() = default; - /// Warning: takes ownership of fd, so closes it in destructor. + /// Warning: takes ownership of fd, so closes it in destructor. inline TFileHandle(FHANDLE fd) noexcept : Fd_(fd) { @@ -147,7 +147,7 @@ private: class TFile { public: TFile(); - /// Takes ownership of handle, so closes it when the last holder of descriptor dies. + /// Takes ownership of handle, so closes it when the last holder of descriptor dies. explicit TFile(FHANDLE fd); TFile(FHANDLE fd, const TString& fname); TFile(const TString& fName, EOpenMode oMode); diff --git a/util/system/file_ut.cpp b/util/system/file_ut.cpp index bb97a6e5d4..941e6a50f3 100644 --- a/util/system/file_ut.cpp +++ b/util/system/file_ut.cpp @@ -408,9 +408,9 @@ UNIT_ASSERT_VALUES_EQUAL(file.CountCache(0, 12345), -1); Y_UNIT_TEST_SUITE(TTestDecodeOpenMode) { Y_UNIT_TEST(It) { UNIT_ASSERT_VALUES_EQUAL("0", DecodeOpenMode(0)); - UNIT_ASSERT_VALUES_EQUAL("RdOnly", DecodeOpenMode(RdOnly)); - UNIT_ASSERT_VALUES_EQUAL("RdWr", DecodeOpenMode(RdWr)); - UNIT_ASSERT_VALUES_EQUAL("WrOnly|ForAppend", DecodeOpenMode(WrOnly | ForAppend)); + UNIT_ASSERT_VALUES_EQUAL("RdOnly", DecodeOpenMode(RdOnly)); + UNIT_ASSERT_VALUES_EQUAL("RdWr", DecodeOpenMode(RdWr)); + UNIT_ASSERT_VALUES_EQUAL("WrOnly|ForAppend", DecodeOpenMode(WrOnly | ForAppend)); UNIT_ASSERT_VALUES_EQUAL("RdWr|CreateAlways|CreateNew|ForAppend|Transient|CloseOnExec|Temp|Sync|Direct|DirectAligned|Seq|NoReuse|NoReadAhead|AX|AR|AW|AWOther|0xF8888000", DecodeOpenMode(0xFFFFFFFF)); - } -} + } +} diff --git a/util/system/filemap.cpp b/util/system/filemap.cpp index 992acdf724..7454a4cb94 100644 --- a/util/system/filemap.cpp +++ b/util/system/filemap.cpp @@ -2,7 +2,7 @@ #include "madvise.h" #include "defaults.h" #include "hi_lo.h" - + #include <util/generic/buffer.h> #include <util/generic/yexception.h> #include <util/generic/singleton.h> diff --git a/util/system/info.cpp b/util/system/info.cpp index 9175c664b3..cf6681e89a 100644 --- a/util/system/info.cpp +++ b/util/system/info.cpp @@ -12,7 +12,7 @@ #if defined(_win_) #include "winint.h" #include <stdio.h> -#else +#else #include <unistd.h> #endif @@ -175,17 +175,17 @@ size_t NSystemInfo::CachedNumberOfCpus() { return NCpus; } - + size_t NSystemInfo::GetPageSize() noexcept { #if defined(_win_) SYSTEM_INFO sysInfo; GetSystemInfo(&sysInfo); - + return sysInfo.dwPageSize; #else return sysconf(_SC_PAGESIZE); #endif -} +} size_t NSystemInfo::TotalMemorySize() { #if defined(_linux_) && defined(_64_) diff --git a/util/system/info_ut.cpp b/util/system/info_ut.cpp index 6923c63005..ad7449f8f4 100644 --- a/util/system/info_ut.cpp +++ b/util/system/info_ut.cpp @@ -7,7 +7,7 @@ class TSysInfoTest: public TTestBase { UNIT_TEST(TestNumberOfCpus) UNIT_TEST(TestGetPageSize) UNIT_TEST_SUITE_END(); - + private: inline void TestNumberOfCpus() { UNIT_ASSERT(NSystemInfo::NumberOfCpus() > 0); diff --git a/util/system/pipe.cpp b/util/system/pipe.cpp index f1d80a363b..a543bd7472 100644 --- a/util/system/pipe.cpp +++ b/util/system/pipe.cpp @@ -2,36 +2,36 @@ #include <util/stream/output.h> #include <util/generic/yexception.h> - + ssize_t TPipeHandle::Read(void* buffer, size_t byteCount) const noexcept { -#ifdef _win_ +#ifdef _win_ return recv(Fd_, (char*)buffer, byteCount, 0); -#else +#else return read(Fd_, buffer, byteCount); -#endif -} - +#endif +} + ssize_t TPipeHandle::Write(const void* buffer, size_t byteCount) const noexcept { -#ifdef _win_ +#ifdef _win_ return send(Fd_, (const char*)buffer, byteCount, 0); -#else +#else return write(Fd_, buffer, byteCount); -#endif -} - +#endif +} + bool TPipeHandle::Close() noexcept { - bool ok = true; + bool ok = true; if (Fd_ != INVALID_PIPEHANDLE) { -#ifdef _win_ +#ifdef _win_ ok = closesocket(Fd_) == 0; -#else +#else ok = close(Fd_) == 0; -#endif - } +#endif + } Fd_ = INVALID_PIPEHANDLE; - return ok; -} - + return ok; +} + void TPipeHandle::Pipe(TPipeHandle& reader, TPipeHandle& writer, EOpenMode mode) { PIPEHANDLE fds[2]; #ifdef _win_ @@ -66,90 +66,90 @@ void TPipeHandle::Pipe(TPipeHandle& reader, TPipeHandle& writer, EOpenMode mode) } class TPipe::TImpl: public TAtomicRefCount<TImpl> { -public: - TImpl() +public: + TImpl() : Handle_(INVALID_PIPEHANDLE) - { - } + { + } - TImpl(PIPEHANDLE fd) + TImpl(PIPEHANDLE fd) : Handle_(fd) - { - } - - inline ~TImpl() { - Close(); - } - - bool IsOpen() { + { + } + + inline ~TImpl() { + Close(); + } + + bool IsOpen() { return Handle_.IsOpen(); - } - - inline void Close() { + } + + inline void Close() { if (!Handle_.IsOpen()) { - return; + return; } if (!Handle_.Close()) { ythrow TFileError() << "failed to close pipe"; } - } - + } + TPipeHandle& GetHandle() noexcept { return Handle_; - } - + } + size_t Read(void* buffer, size_t count) const { ssize_t r = Handle_.Read(buffer, count); if (r < 0) { ythrow TFileError() << "failed to read from pipe"; } - return r; - } - + return r; + } + size_t Write(const void* buffer, size_t count) const { ssize_t r = Handle_.Write(buffer, count); if (r < 0) { ythrow TFileError() << "failed to write to pipe"; } - return r; - } + return r; + } -private: +private: TPipeHandle Handle_; -}; - -TPipe::TPipe() +}; + +TPipe::TPipe() : Impl_(new TImpl) -{ -} - -TPipe::TPipe(PIPEHANDLE fd) +{ +} + +TPipe::TPipe(PIPEHANDLE fd) : Impl_(new TImpl(fd)) -{ -} - +{ +} + TPipe::~TPipe() = default; - -void TPipe::Close() { + +void TPipe::Close() { Impl_->Close(); -} - +} + PIPEHANDLE TPipe::GetHandle() const noexcept { return Impl_->GetHandle(); -} - +} + bool TPipe::IsOpen() const noexcept { return Impl_->IsOpen(); -} - +} + size_t TPipe::Read(void* buf, size_t len) const { return Impl_->Read(buf, len); -} - +} + size_t TPipe::Write(const void* buf, size_t len) const { return Impl_->Write(buf, len); -} - +} + void TPipe::Pipe(TPipe& reader, TPipe& writer, EOpenMode mode) { TImplRef r(new TImpl()); TImplRef w(new TImpl()); @@ -158,4 +158,4 @@ void TPipe::Pipe(TPipe& reader, TPipe& writer, EOpenMode mode) { r.Swap(reader.Impl_); w.Swap(writer.Impl_); -} +} diff --git a/util/system/pipe.h b/util/system/pipe.h index b8cd9bd2ac..75d0360049 100644 --- a/util/system/pipe.h +++ b/util/system/pipe.h @@ -4,19 +4,19 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Warray-bounds" //need because of bug in gcc4.9.2 #endif - + #include "defaults.h" #include "file.h" -#include <util/generic/ptr.h> +#include <util/generic/ptr.h> #include <util/network/pair.h> #include <util/generic/noncopyable.h> - + using PIPEHANDLE = SOCKET; #define INVALID_PIPEHANDLE INVALID_SOCKET -/// Pipe-like object: pipe on POSIX and socket on windows +/// Pipe-like object: pipe on POSIX and socket on windows class TPipeHandle: public TNonCopyable { -public: +public: inline TPipeHandle() noexcept : Fd_(INVALID_PIPEHANDLE) { @@ -24,66 +24,66 @@ public: inline TPipeHandle(PIPEHANDLE fd) noexcept : Fd_(fd) - { - } - + { + } + inline ~TPipeHandle() { - Close(); - } - + Close(); + } + bool Close() noexcept; - + inline PIPEHANDLE Release() noexcept { PIPEHANDLE ret = Fd_; Fd_ = INVALID_PIPEHANDLE; - return ret; - } - + return ret; + } + inline void Swap(TPipeHandle& r) noexcept { DoSwap(Fd_, r.Fd_); - } - + } + inline operator PIPEHANDLE() const noexcept { return Fd_; - } - + } + inline bool IsOpen() const noexcept { return Fd_ != INVALID_PIPEHANDLE; - } - + } + ssize_t Read(void* buffer, size_t byteCount) const noexcept; ssize_t Write(const void* buffer, size_t byteCount) const noexcept; - + // Only CloseOnExec is supported static void Pipe(TPipeHandle& reader, TPipeHandle& writer, EOpenMode mode = 0); -private: +private: PIPEHANDLE Fd_; -}; - -class TPipe { -public: - TPipe(); - /// Takes ownership of handle, so closes it when the last holder of descriptor dies. - explicit TPipe(PIPEHANDLE fd); +}; + +class TPipe { +public: + TPipe(); + /// Takes ownership of handle, so closes it when the last holder of descriptor dies. + explicit TPipe(PIPEHANDLE fd); ~TPipe(); - - void Close(); - + + void Close(); + bool IsOpen() const noexcept; PIPEHANDLE GetHandle() const noexcept; - + size_t Read(void* buf, size_t len) const; size_t Write(const void* buf, size_t len) const; - + // Only CloseOnExec is supported static void Pipe(TPipe& reader, TPipe& writer, EOpenMode mode = 0); - -private: - class TImpl; + +private: + class TImpl; using TImplRef = TSimpleIntrusivePtr<TImpl>; TImplRef Impl_; -}; +}; #ifdef __GNUC__ #pragma GCC diagnostic pop diff --git a/util/system/pipe_ut.cpp b/util/system/pipe_ut.cpp index 67cc0eb5f0..6d53432de8 100644 --- a/util/system/pipe_ut.cpp +++ b/util/system/pipe_ut.cpp @@ -1,15 +1,15 @@ #include "pipe.h" #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TPipeTest) { Y_UNIT_TEST(TestPipe) { - TPipe r; - TPipe w; - TPipe::Pipe(r, w); - char c = 'a'; - UNIT_ASSERT(1 == w.Write(&c, 1)); - UNIT_ASSERT(1 == r.Read(&c, 1)); - UNIT_ASSERT_VALUES_EQUAL('a', c); - } -} + TPipe r; + TPipe w; + TPipe::Pipe(r, w); + char c = 'a'; + UNIT_ASSERT(1 == w.Write(&c, 1)); + UNIT_ASSERT(1 == r.Read(&c, 1)); + UNIT_ASSERT_VALUES_EQUAL('a', c); + } +} diff --git a/util/system/rusage.cpp b/util/system/rusage.cpp index 918169ebdf..2befeca875 100644 --- a/util/system/rusage.cpp +++ b/util/system/rusage.cpp @@ -1,40 +1,40 @@ #include "platform.h" - + #if defined(__APPLE__) && defined(__MACH__) #include <mach/mach.h> #endif -#ifdef _win_ - +#ifdef _win_ + #include "winint.h" #include <psapi.h> - -#else - + +#else + #include <sys/time.h> #include <sys/resource.h> - -#endif - -#include <util/generic/yexception.h> - -#include "info.h" - -#include "rusage.h" - -#ifdef _win_ -TDuration FiletimeToDuration(const FILETIME& ft) { + +#endif + +#include <util/generic/yexception.h> + +#include "info.h" + +#include "rusage.h" + +#ifdef _win_ +TDuration FiletimeToDuration(const FILETIME& ft) { union { ui64 ft_scalar; FILETIME ft_struct; } nt_time; - nt_time.ft_struct = ft; - return TDuration::MicroSeconds(nt_time.ft_scalar / 10); -} -#endif - + nt_time.ft_struct = ft; + return TDuration::MicroSeconds(nt_time.ft_scalar / 10); +} +#endif + size_t TRusage::GetCurrentRSS() { /* * Author: David Robert Nadeau @@ -74,40 +74,40 @@ size_t TRusage::GetCurrentRSS() { #endif } -void TRusage::Fill() { - *this = TRusage(); - -#ifdef _win_ - // copy-paste from PostgreSQL getrusage.c - - FILETIME starttime; - FILETIME exittime; - FILETIME kerneltime; - FILETIME usertime; - - if (GetProcessTimes(GetCurrentProcess(), &starttime, &exittime, &kerneltime, &usertime) == 0) { - ythrow TSystemError() << "GetProcessTimes failed"; - } - - Utime = FiletimeToDuration(usertime); - Stime = FiletimeToDuration(kerneltime); - - PROCESS_MEMORY_COUNTERS pmc; - - if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) == 0) { - ythrow TSystemError() << "GetProcessMemoryInfo failed"; - } - +void TRusage::Fill() { + *this = TRusage(); + +#ifdef _win_ + // copy-paste from PostgreSQL getrusage.c + + FILETIME starttime; + FILETIME exittime; + FILETIME kerneltime; + FILETIME usertime; + + if (GetProcessTimes(GetCurrentProcess(), &starttime, &exittime, &kerneltime, &usertime) == 0) { + ythrow TSystemError() << "GetProcessTimes failed"; + } + + Utime = FiletimeToDuration(usertime); + Stime = FiletimeToDuration(kerneltime); + + PROCESS_MEMORY_COUNTERS pmc; + + if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc)) == 0) { + ythrow TSystemError() << "GetProcessMemoryInfo failed"; + } + MaxRss = pmc.PeakWorkingSetSize; MajorPageFaults = pmc.PageFaultCount; - -#else - struct rusage ru; - int r = getrusage(RUSAGE_SELF, &ru); - if (r < 0) { - ythrow TSystemError() << "rusage failed"; - } - + +#else + struct rusage ru; + int r = getrusage(RUSAGE_SELF, &ru); + if (r < 0) { + ythrow TSystemError() << "rusage failed"; + } + #if defined(_darwin_) // see https://lists.apple.com/archives/darwin-kernel/2009/Mar/msg00005.html MaxRss = ru.ru_maxrss; @@ -115,7 +115,7 @@ void TRusage::Fill() { MaxRss = ru.ru_maxrss * 1024LL; #endif MajorPageFaults = ru.ru_majflt; - Utime = ru.ru_utime; - Stime = ru.ru_stime; -#endif -} + Utime = ru.ru_utime; + Stime = ru.ru_stime; +#endif +} diff --git a/util/system/rusage.h b/util/system/rusage.h index 3964df1116..61aeca83f2 100644 --- a/util/system/rusage.h +++ b/util/system/rusage.h @@ -1,26 +1,26 @@ #pragma once - + #include "defaults.h" -#include <util/generic/utility.h> -#include <util/datetime/base.h> - -/// portable getrusage - -struct TRusage { - // some fields may be zero if unsupported - +#include <util/generic/utility.h> +#include <util/datetime/base.h> + +/// portable getrusage + +struct TRusage { + // some fields may be zero if unsupported + ui64 MaxRss = 0; ui64 MajorPageFaults = 0; - TDuration Utime; - TDuration Stime; - - void Fill(); + TDuration Utime; + TDuration Stime; + + void Fill(); static size_t GetCurrentRSS(); - static TRusage Get() { - TRusage r; - r.Fill(); - return r; - } -}; + static TRusage Get() { + TRusage r; + r.Fill(); + return r; + } +}; diff --git a/util/system/rusage_ut.cpp b/util/system/rusage_ut.cpp index a5c6a80e62..0d4e0fe54b 100644 --- a/util/system/rusage_ut.cpp +++ b/util/system/rusage_ut.cpp @@ -1,11 +1,11 @@ #include "rusage.h" #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TRusageTest) { Y_UNIT_TEST(TestRusage) { - TRusage r; - // just check it returns something - r.Fill(); - } -} + TRusage r; + // just check it returns something + r.Fill(); + } +} diff --git a/util/system/spin_wait.h b/util/system/spin_wait.h index e52f413e6d..91dd423e33 100644 --- a/util/system/spin_wait.h +++ b/util/system/spin_wait.h @@ -4,7 +4,7 @@ struct TSpinWait { TSpinWait() noexcept; void Sleep() noexcept; - + unsigned T; unsigned C; -}; +}; diff --git a/util/system/spinlock.h b/util/system/spinlock.h index 9d1480130b..af2630890a 100644 --- a/util/system/spinlock.h +++ b/util/system/spinlock.h @@ -3,29 +3,29 @@ #include "atomic.h" #include "spin_wait.h" -class TSpinLockBase { +class TSpinLockBase { protected: inline TSpinLockBase() noexcept { AtomicSet(Val_, 0); } - + public: inline bool IsLocked() const noexcept { return AtomicGet(Val_); } - + inline bool TryAcquire() noexcept { return AtomicTryLock(&Val_); } - + inline bool try_lock() noexcept { return TryAcquire(); } protected: TAtomic Val_; -}; - +}; + static inline void SpinLockPause() { #if defined(__GNUC__) #if defined(_i386_) || defined(_x86_64_) diff --git a/util/system/spinlock_ut.cpp b/util/system/spinlock_ut.cpp index f2793fd95f..e8639a6404 100644 --- a/util/system/spinlock_ut.cpp +++ b/util/system/spinlock_ut.cpp @@ -1,23 +1,23 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "spinlock.h" - + +#include "spinlock.h" + Y_UNIT_TEST_SUITE(TSpinLock) { - template <typename TLock> - void TestLock() { - TLock lock; - UNIT_ASSERT(!lock.IsLocked()); - lock.Acquire(); - UNIT_ASSERT(lock.IsLocked()); - lock.Release(); - UNIT_ASSERT(!lock.IsLocked()); - - UNIT_ASSERT(lock.TryAcquire()); - UNIT_ASSERT(lock.IsLocked()); - UNIT_ASSERT(!lock.TryAcquire()); - UNIT_ASSERT(lock.IsLocked()); - lock.Release(); - UNIT_ASSERT(!lock.IsLocked()); + template <typename TLock> + void TestLock() { + TLock lock; + UNIT_ASSERT(!lock.IsLocked()); + lock.Acquire(); + UNIT_ASSERT(lock.IsLocked()); + lock.Release(); + UNIT_ASSERT(!lock.IsLocked()); + + UNIT_ASSERT(lock.TryAcquire()); + UNIT_ASSERT(lock.IsLocked()); + UNIT_ASSERT(!lock.TryAcquire()); + UNIT_ASSERT(lock.IsLocked()); + lock.Release(); + UNIT_ASSERT(!lock.IsLocked()); // Lockable requirements lock.lock(); @@ -25,13 +25,13 @@ Y_UNIT_TEST_SUITE(TSpinLock) { UNIT_ASSERT(!lock.try_lock()); lock.unlock(); UNIT_ASSERT(!lock.IsLocked()); - } - + } + Y_UNIT_TEST(TSpinLock_IsLocked) { - TestLock<TSpinLock>(); - } - + TestLock<TSpinLock>(); + } + Y_UNIT_TEST(TAdaptiveLock_IsLocked) { - TestLock<TAdaptiveLock>(); - } -} + TestLock<TAdaptiveLock>(); + } +} diff --git a/util/system/thread.cpp b/util/system/thread.cpp index 0aad030bcf..6236746c2d 100644 --- a/util/system/thread.cpp +++ b/util/system/thread.cpp @@ -5,7 +5,7 @@ #include <util/generic/ptr.h> #include <util/generic/ymath.h> #include <util/generic/ylimits.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include "yassert.h" #include <utility> @@ -289,7 +289,7 @@ TThread::~TThread() { Join(); } -void TThread::Start() { +void TThread::Start() { Impl(Impl_, "start", false)->Start(); } diff --git a/util/system/thread.h b/util/system/thread.h index 541ecccbef..a6e8abdb5b 100644 --- a/util/system/thread.h +++ b/util/system/thread.h @@ -1,9 +1,9 @@ #pragma once -/// This code should not be used directly unless you really understand what you do. +/// This code should not be used directly unless you really understand what you do. /// If you need threads, use thread pool functionality in <util/thread/factory.h> /// @see SystemThreadFactory() - + #include <util/generic/ptr.h> #include <util/generic/string.h> diff --git a/util/system/type_name.cpp b/util/system/type_name.cpp index 411b98774c..0377da4212 100644 --- a/util/system/type_name.cpp +++ b/util/system/type_name.cpp @@ -1,13 +1,13 @@ #include "platform.h" #include "demangle_impl.h" - + #ifdef __GNUC__ #include <stdexcept> #include <cxxabi.h> -#endif - +#endif + #include "type_name.h" - + namespace { #if defined(_LIBCPP_VERSION) @@ -26,18 +26,18 @@ namespace { const char* NPrivate::TCppDemangler::Demangle(const char* name) { #ifndef __GNUC__ - return name; -#else - int status; + return name; +#else + int status; TmpBuf_.Reset(__cxxabiv1::__cxa_demangle(name, nullptr, nullptr, &status)); if (!TmpBuf_) { - return name; + return name; } return TmpBuf_.Get(); -#endif -} +#endif +} TString CppDemangle(const TString& name) { return NPrivate::TCppDemangler().Demangle(name.data()); diff --git a/util/system/type_name.h b/util/system/type_name.h index 6561519bc1..b6619aba3f 100644 --- a/util/system/type_name.h +++ b/util/system/type_name.h @@ -1,8 +1,8 @@ #pragma once - + #include <util/generic/string.h> #include <util/string/subst.h> - + #include <typeindex> #include <typeinfo> diff --git a/util/system/type_name_ut.cpp b/util/system/type_name_ut.cpp index 7af6838814..86597f4232 100644 --- a/util/system/type_name_ut.cpp +++ b/util/system/type_name_ut.cpp @@ -1,7 +1,7 @@ #include "type_name.h" #include <library/cpp/testing/unittest/registar.h> - + #include <util/generic/yexception.h> #include <util/generic/fwd.h> @@ -10,12 +10,12 @@ Y_UNIT_TEST_SUITE(TDemangleTest) { Y_UNIT_TEST(SimpleTest) { - // just check it does not crash or leak - CppDemangle("hello"); - CppDemangle(""); - CppDemangle("Sfsdf$dfsdfTTSFSDF23234::SDFS:FSDFSDF#$%"); - } -} + // just check it does not crash or leak + CppDemangle("hello"); + CppDemangle(""); + CppDemangle("Sfsdf$dfsdfTTSFSDF23234::SDFS:FSDFSDF#$%"); + } +} namespace NUtil::NTypeNameTest { diff --git a/util/system/yassert.cpp b/util/system/yassert.cpp index d356f11300..0f586648b7 100644 --- a/util/system/yassert.cpp +++ b/util/system/yassert.cpp @@ -12,7 +12,7 @@ #include <util/stream/output.h> #include <util/stream/str.h> #include <util/string/printf.h> - + #include <cstdlib> #include <stdarg.h> #include <stdio.h> @@ -37,20 +37,20 @@ namespace { namespace NPrivate { [[noreturn]] Y_NO_INLINE void InternalPanicImpl(int line, const char* function, const char* expr, int, int, int, const TStringBuf file, const char* errorMessage, size_t errorMessageSize) noexcept; } - + void ::NPrivate::Panic(const TStaticBuf& file, int line, const char* function, const char* expr, const char* format, ...) noexcept { try { // We care of panic of first failed thread only // Otherwise stderr could contain multiple messages and stack traces shuffled auto guard = Guard(*Singleton<TPanicLockHolder>()); - + TString errorMsg; va_list args; va_start(args, format); - // format has " " prefix to mute GCC warning on empty format - vsprintf(errorMsg, format[0] == ' ' ? format + 1 : format, args); + // format has " " prefix to mute GCC warning on empty format + vsprintf(errorMsg, format[0] == ' ' ? format + 1 : format, args); va_end(args); - + constexpr int abiPlaceholder = 0; ::NPrivate::InternalPanicImpl(line, function, expr, abiPlaceholder, abiPlaceholder, abiPlaceholder, file.As<TStringBuf>(), errorMsg.c_str(), errorMsg.size()); } catch (...) { @@ -69,19 +69,19 @@ namespace NPrivate { TStringOutput o(r); if (expr) { o << "VERIFY failed (" << now << "): " << errorMsg << Endl; - } else { + } else { o << "FAIL (" << now << "): " << errorMsg << Endl; - } + } o << " " << file << ":" << line << Endl; if (expr) { - o << " " << function << "(): requirement " << expr << " failed" << Endl; - } else { - o << " " << function << "() failed" << Endl; - } + o << " " << function << "(): requirement " << expr << " failed" << Endl; + } else { + o << " " << function << "() failed" << Endl; + } Cerr << r << Flush; -#ifndef WITH_VALGRIND +#ifndef WITH_VALGRIND PrintBackTrace(); -#endif +#endif #ifdef CLANG_COVERAGE if (__llvm_profile_write_file()) { Cerr << "Failed to dump clang coverage" << Endl; @@ -91,4 +91,4 @@ namespace NPrivate { } catch (...) { abort(); } -} +} diff --git a/util/system/yassert.h b/util/system/yassert.h index a94b47d3f4..529823440c 100644 --- a/util/system/yassert.h +++ b/util/system/yassert.h @@ -3,7 +3,7 @@ #include "defaults.h" #include "src_root.h" #include "backtrace.h" - + #if defined(_MSC_VER) #include <new> #if defined(_DEBUG) @@ -89,12 +89,12 @@ inline void YaDebugBreak() { } while (false) #endif -namespace NPrivate { - /// method should not be used directly +namespace NPrivate { + /// method should not be used directly [[noreturn]] void Panic(const TStaticBuf& file, int line, const char* function, const char* expr, const char* format, ...) noexcept Y_PRINTF_FORMAT(5, 6); -} - -/// Assert that does not depend on NDEBUG macro and outputs message like printf +} + +/// Assert that does not depend on NDEBUG macro and outputs message like printf #define Y_VERIFY(expr, ...) \ do { \ if (Y_UNLIKELY(!(expr))) { \ @@ -106,7 +106,7 @@ namespace NPrivate { do { \ ::NPrivate::Panic(__SOURCE_FILE_IMPL__, __LINE__, __FUNCTION__, nullptr, " " __VA_ARGS__); \ } while (false) - + #ifndef NDEBUG /// Assert that depend on NDEBUG macro and outputs message like printf #define Y_VERIFY_DEBUG(expr, ...) \ diff --git a/util/system/yassert_ut.cpp b/util/system/yassert_ut.cpp index c3f8cce999..ddd392666c 100644 --- a/util/system/yassert_ut.cpp +++ b/util/system/yassert_ut.cpp @@ -1,9 +1,9 @@ -#undef NDEBUG -// yassert.h must be included before all headers -#include "yassert.h" - +#undef NDEBUG +// yassert.h must be included before all headers +#include "yassert.h" + #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(YassertTest) { Y_UNIT_TEST(TestAcsLikeFunctionCall) { if (true) { @@ -11,25 +11,25 @@ Y_UNIT_TEST_SUITE(YassertTest) { } else { Y_ASSERT(false); } - - bool var = false; + + bool var = false; if (false) { Y_ASSERT(false); } else { var = true; // this is unreachable if Y_ASSERT is "if (!cond) { ... }" } - UNIT_ASSERT(var); - } - + UNIT_ASSERT(var); + } + Y_UNIT_TEST(TestFailCompiles) { - if (false) { + if (false) { Y_FAIL("%d is a lucky number", 7); Y_FAIL(); - } - } - + } + } + Y_UNIT_TEST(TestVerify) { Y_VERIFY(true, "hi %s", "there"); Y_VERIFY(true); - } -} + } +} diff --git a/util/thread/lfqueue.h b/util/thread/lfqueue.h index 2bd4416737..ab523631e4 100644 --- a/util/thread/lfqueue.h +++ b/util/thread/lfqueue.h @@ -173,27 +173,27 @@ class TLockFreeQueue: public TNonCopyable { void EnqueueImpl(TListNode* head, TListNode* tail) { TRootNode* newRoot = new TRootNode; - AsyncRef(); + AsyncRef(); AtomicSet(newRoot->PushQueue, head); - for (;;) { + for (;;) { TRootNode* curRoot = AtomicGet(JobQueue); AtomicSet(tail->Next, AtomicGet(curRoot->PushQueue)); AtomicSet(newRoot->PopQueue, AtomicGet(curRoot->PopQueue)); - newRoot->CopyCounter(curRoot); - + newRoot->CopyCounter(curRoot); + for (TListNode* node = head;; node = AtomicGet(node->Next)) { - newRoot->IncCount(node->Data); - if (node == tail) - break; - } - - if (AtomicCas(&JobQueue, newRoot, curRoot)) { + newRoot->IncCount(node->Data); + if (node == tail) + break; + } + + if (AtomicCas(&JobQueue, newRoot, curRoot)) { AsyncUnref(curRoot, nullptr); - break; - } - } - } - + break; + } + } + } + template <typename TCollection> static void FillCollection(TListNode* lst, TCollection* res) { while (lst) { @@ -242,8 +242,8 @@ public: template <typename U> void Enqueue(U&& data) { TListNode* newNode = new TListNode(std::forward<U>(data)); - EnqueueImpl(newNode, newNode); - } + EnqueueImpl(newNode, newNode); + } void Enqueue(T&& data) { TListNode* newNode = new TListNode(std::move(data)); EnqueueImpl(newNode, newNode); @@ -252,24 +252,24 @@ public: TListNode* newNode = new TListNode(data); EnqueueImpl(newNode, newNode); } - template <typename TCollection> + template <typename TCollection> void EnqueueAll(const TCollection& data) { - EnqueueAll(data.begin(), data.end()); - } - template <typename TIter> + EnqueueAll(data.begin(), data.end()); + } + template <typename TIter> void EnqueueAll(TIter dataBegin, TIter dataEnd) { - if (dataBegin == dataEnd) - return; - - TIter i = dataBegin; + if (dataBegin == dataEnd) + return; + + TIter i = dataBegin; TListNode* volatile node = new TListNode(*i); TListNode* volatile tail = node; - - for (++i; i != dataEnd; ++i) { - TListNode* nextNode = node; + + for (++i; i != dataEnd; ++i) { + TListNode* nextNode = node; node = new TListNode(*i, nextNode); } - EnqueueImpl(node, tail); + EnqueueImpl(node, tail); } bool Dequeue(T* data) { TRootNode* newRoot = nullptr; diff --git a/util/thread/lfqueue_ut.cpp b/util/thread/lfqueue_ut.cpp index d65fbd3769..83bca100cf 100644 --- a/util/thread/lfqueue_ut.cpp +++ b/util/thread/lfqueue_ut.cpp @@ -6,9 +6,9 @@ #include <util/generic/ptr.h> #include <util/system/atomic.h> #include <util/thread/pool.h> - + #include "lfqueue.h" - + class TMoveTest { public: TMoveTest(int marker = 0, int value = 0) @@ -122,74 +122,74 @@ Y_UNIT_TEST_SUITE(TLockFreeQueueTests) { } Y_UNIT_TEST(TestSimpleEnqueueDequeue) { - TLockFreeQueue<int> queue; - + TLockFreeQueue<int> queue; + int i = -1; - - UNIT_ASSERT(!queue.Dequeue(&i)); + + UNIT_ASSERT(!queue.Dequeue(&i)); UNIT_ASSERT_VALUES_EQUAL(i, -1); - - queue.Enqueue(10); - queue.Enqueue(11); - queue.Enqueue(12); - - UNIT_ASSERT(queue.Dequeue(&i)); - UNIT_ASSERT_VALUES_EQUAL(10, i); - UNIT_ASSERT(queue.Dequeue(&i)); - UNIT_ASSERT_VALUES_EQUAL(11, i); - - queue.Enqueue(13); - - UNIT_ASSERT(queue.Dequeue(&i)); - UNIT_ASSERT_VALUES_EQUAL(12, i); - UNIT_ASSERT(queue.Dequeue(&i)); - UNIT_ASSERT_VALUES_EQUAL(13, i); - - UNIT_ASSERT(!queue.Dequeue(&i)); + + queue.Enqueue(10); + queue.Enqueue(11); + queue.Enqueue(12); + + UNIT_ASSERT(queue.Dequeue(&i)); + UNIT_ASSERT_VALUES_EQUAL(10, i); + UNIT_ASSERT(queue.Dequeue(&i)); + UNIT_ASSERT_VALUES_EQUAL(11, i); + + queue.Enqueue(13); + + UNIT_ASSERT(queue.Dequeue(&i)); + UNIT_ASSERT_VALUES_EQUAL(12, i); + UNIT_ASSERT(queue.Dequeue(&i)); + UNIT_ASSERT_VALUES_EQUAL(13, i); + + UNIT_ASSERT(!queue.Dequeue(&i)); const int tmp = 100; queue.Enqueue(tmp); UNIT_ASSERT(queue.Dequeue(&i)); UNIT_ASSERT_VALUES_EQUAL(i, tmp); - } - + } + Y_UNIT_TEST(TestSimpleEnqueueAllDequeue) { - TLockFreeQueue<int> queue; - + TLockFreeQueue<int> queue; + int i = -1; - - UNIT_ASSERT(!queue.Dequeue(&i)); + + UNIT_ASSERT(!queue.Dequeue(&i)); UNIT_ASSERT_VALUES_EQUAL(i, -1); - + TVector<int> v; - v.push_back(20); - v.push_back(21); - - queue.EnqueueAll(v); - - v.clear(); - v.push_back(22); - v.push_back(23); - v.push_back(24); - - queue.EnqueueAll(v); - - v.clear(); - queue.EnqueueAll(v); - - v.clear(); - v.push_back(25); - - queue.EnqueueAll(v); - - for (int j = 20; j <= 25; ++j) { - UNIT_ASSERT(queue.Dequeue(&i)); - UNIT_ASSERT_VALUES_EQUAL(j, i); - } - - UNIT_ASSERT(!queue.Dequeue(&i)); - } - + v.push_back(20); + v.push_back(21); + + queue.EnqueueAll(v); + + v.clear(); + v.push_back(22); + v.push_back(23); + v.push_back(24); + + queue.EnqueueAll(v); + + v.clear(); + queue.EnqueueAll(v); + + v.clear(); + v.push_back(25); + + queue.EnqueueAll(v); + + for (int j = 20; j <= 25; ++j) { + UNIT_ASSERT(queue.Dequeue(&i)); + UNIT_ASSERT_VALUES_EQUAL(j, i); + } + + UNIT_ASSERT(!queue.Dequeue(&i)); + } + void DequeueAllRunner(TLockFreeQueue<int>& queue, bool singleConsumer) { size_t threadsNum = 4; size_t enqueuesPerThread = 10'000; @@ -304,19 +304,19 @@ Y_UNIT_TEST_SUITE(TLockFreeQueueTests) { Y_UNIT_TEST(CleanInDestructor) { TSimpleSharedPtr<bool> p(new bool); - UNIT_ASSERT_VALUES_EQUAL(1u, p.RefCount()); - - { + UNIT_ASSERT_VALUES_EQUAL(1u, p.RefCount()); + + { TLockFreeQueue<TSimpleSharedPtr<bool>> stack; - - stack.Enqueue(p); - stack.Enqueue(p); - - UNIT_ASSERT_VALUES_EQUAL(3u, p.RefCount()); - } - - UNIT_ASSERT_VALUES_EQUAL(1, p.RefCount()); - } + + stack.Enqueue(p); + stack.Enqueue(p); + + UNIT_ASSERT_VALUES_EQUAL(3u, p.RefCount()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, p.RefCount()); + } Y_UNIT_TEST(CheckOperationsCount) { TOperationsChecker o; @@ -330,4 +330,4 @@ Y_UNIT_TEST_SUITE(TLockFreeQueueTests) { queue.Dequeue(&o); o.Check(0, 0, 2, 1, 0); } -} +} diff --git a/util/thread/lfstack.h b/util/thread/lfstack.h index 81534cd1d1..ca3d95f3c3 100644 --- a/util/thread/lfstack.h +++ b/util/thread/lfstack.h @@ -43,12 +43,12 @@ class TLockFreeStack: TNonCopyable { } } void EnqueueImpl(TNode* volatile head, TNode* volatile tail) { - for (;;) { + for (;;) { tail->Next = AtomicGet(Head); - if (AtomicCas(&Head, head, tail->Next)) - break; - } - } + if (AtomicCas(&Head, head, tail->Next)) + break; + } + } template <class U> void EnqueueImpl(U&& u) { TNode* volatile node = new TNode(std::forward<U>(u)); @@ -69,31 +69,31 @@ public: void Enqueue(const T& t) { EnqueueImpl(t); - } + } void Enqueue(T&& t) { EnqueueImpl(std::move(t)); } - template <typename TCollection> + template <typename TCollection> void EnqueueAll(const TCollection& data) { - EnqueueAll(data.begin(), data.end()); - } - template <typename TIter> + EnqueueAll(data.begin(), data.end()); + } + template <typename TIter> void EnqueueAll(TIter dataBegin, TIter dataEnd) { - if (dataBegin == dataEnd) { - return; + if (dataBegin == dataEnd) { + return; } - TIter i = dataBegin; + TIter i = dataBegin; TNode* volatile node = new TNode(*i); TNode* volatile tail = node; - - for (++i; i != dataEnd; ++i) { - TNode* nextNode = node; - node = new TNode(*i); - node->Next = nextNode; - } - EnqueueImpl(node, tail); + + for (++i; i != dataEnd; ++i) { + TNode* nextNode = node; + node = new TNode(*i); + node->Next = nextNode; + } + EnqueueImpl(node, tail); } bool Dequeue(T* res) { AtomicAdd(DequeueCount, 1); @@ -122,8 +122,8 @@ public: return false; } // add all elements to *res - // elements are returned in order of dequeue (top to bottom; see example in unittest) - template <typename TCollection> + // elements are returned in order of dequeue (top to bottom; see example in unittest) + template <typename TCollection> void DequeueAll(TCollection* res) { AtomicAdd(DequeueCount, 1); for (TNode* current = AtomicGet(Head); current; current = AtomicGet(Head)) { @@ -168,7 +168,7 @@ public: } // add all elements to *res // elements are returned in order of dequeue (top to bottom; see example in unittest) - template <typename TCollection> + template <typename TCollection> void DequeueAllSingleConsumer(TCollection* res) { for (TNode* current = AtomicGet(Head); current; current = AtomicGet(Head)) { if (AtomicCas(&Head, (TNode*)nullptr, current)) { diff --git a/util/thread/lfstack_ut.cpp b/util/thread/lfstack_ut.cpp index 824d633a62..e20a838f95 100644 --- a/util/thread/lfstack_ut.cpp +++ b/util/thread/lfstack_ut.cpp @@ -1,194 +1,194 @@ -#include <util/system/atomic.h> -#include <util/system/event.h> +#include <util/system/atomic.h> +#include <util/system/event.h> #include <util/generic/deque.h> #include <library/cpp/threading/future/legacy_future.h> - + #include <library/cpp/testing/unittest/registar.h> - + #include "lfstack.h" - + Y_UNIT_TEST_SUITE(TLockFreeStackTests) { - class TCountDownLatch { - private: + class TCountDownLatch { + private: TAtomic Current_; TSystemEvent EventObject_; - public: - TCountDownLatch(unsigned initial) + public: + TCountDownLatch(unsigned initial) : Current_(initial) { } - - void CountDown() { + + void CountDown() { if (AtomicDecrement(Current_) == 0) { EventObject_.Signal(); - } - } - - void Await() { + } + } + + void Await() { EventObject_.Wait(); - } - - bool Await(TDuration timeout) { + } + + bool Await(TDuration timeout) { return EventObject_.WaitT(timeout); - } - }; - - template <bool SingleConsumer> - struct TDequeueAllTester { - size_t EnqueueThreads; - size_t DequeueThreads; - - size_t EnqueuesPerThread; - TAtomic LeftToDequeue; - - TCountDownLatch StartLatch; - TLockFreeStack<int> Stack; - - TDequeueAllTester() - : EnqueueThreads(4) - , DequeueThreads(SingleConsumer ? 1 : 3) - , EnqueuesPerThread(100000) - , LeftToDequeue(EnqueueThreads * EnqueuesPerThread) - , StartLatch(EnqueueThreads + DequeueThreads) - { - } - - void Enqueuer() { - StartLatch.CountDown(); - StartLatch.Await(); - - for (size_t i = 0; i < EnqueuesPerThread; ++i) { - Stack.Enqueue(i); - } - } - - void DequeuerAll() { - StartLatch.CountDown(); - StartLatch.Await(); - + } + }; + + template <bool SingleConsumer> + struct TDequeueAllTester { + size_t EnqueueThreads; + size_t DequeueThreads; + + size_t EnqueuesPerThread; + TAtomic LeftToDequeue; + + TCountDownLatch StartLatch; + TLockFreeStack<int> Stack; + + TDequeueAllTester() + : EnqueueThreads(4) + , DequeueThreads(SingleConsumer ? 1 : 3) + , EnqueuesPerThread(100000) + , LeftToDequeue(EnqueueThreads * EnqueuesPerThread) + , StartLatch(EnqueueThreads + DequeueThreads) + { + } + + void Enqueuer() { + StartLatch.CountDown(); + StartLatch.Await(); + + for (size_t i = 0; i < EnqueuesPerThread; ++i) { + Stack.Enqueue(i); + } + } + + void DequeuerAll() { + StartLatch.CountDown(); + StartLatch.Await(); + TVector<int> temp; - while (AtomicGet(LeftToDequeue) > 0) { - size_t dequeued = 0; - for (size_t i = 0; i < 100; ++i) { - temp.clear(); - if (SingleConsumer) { - Stack.DequeueAllSingleConsumer(&temp); - } else { - Stack.DequeueAll(&temp); - } - dequeued += temp.size(); - } - AtomicAdd(LeftToDequeue, -dequeued); - } - } - - void Run() { + while (AtomicGet(LeftToDequeue) > 0) { + size_t dequeued = 0; + for (size_t i = 0; i < 100; ++i) { + temp.clear(); + if (SingleConsumer) { + Stack.DequeueAllSingleConsumer(&temp); + } else { + Stack.DequeueAll(&temp); + } + dequeued += temp.size(); + } + AtomicAdd(LeftToDequeue, -dequeued); + } + } + + void Run() { TVector<TSimpleSharedPtr<NThreading::TLegacyFuture<>>> futures; - - for (size_t i = 0; i < EnqueueThreads; ++i) { + + for (size_t i = 0; i < EnqueueThreads; ++i) { futures.push_back(new NThreading::TLegacyFuture<>(std::bind(&TDequeueAllTester<SingleConsumer>::Enqueuer, this))); - } - - for (size_t i = 0; i < DequeueThreads; ++i) { + } + + for (size_t i = 0; i < DequeueThreads; ++i) { futures.push_back(new NThreading::TLegacyFuture<>(std::bind(&TDequeueAllTester<SingleConsumer>::DequeuerAll, this))); - } - - // effectively join - futures.clear(); - - UNIT_ASSERT_VALUES_EQUAL(0, int(AtomicGet(LeftToDequeue))); - + } + + // effectively join + futures.clear(); + + UNIT_ASSERT_VALUES_EQUAL(0, int(AtomicGet(LeftToDequeue))); + TVector<int> left; - Stack.DequeueAll(&left); - UNIT_ASSERT(left.empty()); - } - }; - + Stack.DequeueAll(&left); + UNIT_ASSERT(left.empty()); + } + }; + Y_UNIT_TEST(TestDequeueAll) { - TDequeueAllTester<false>().Run(); - } - + TDequeueAllTester<false>().Run(); + } + Y_UNIT_TEST(TestDequeueAllSingleConsumer) { - TDequeueAllTester<true>().Run(); - } - + TDequeueAllTester<true>().Run(); + } + Y_UNIT_TEST(TestDequeueAllEmptyStack) { - TLockFreeStack<int> stack; - + TLockFreeStack<int> stack; + TVector<int> r; - stack.DequeueAll(&r); - - UNIT_ASSERT(r.empty()); - } - + stack.DequeueAll(&r); + + UNIT_ASSERT(r.empty()); + } + Y_UNIT_TEST(TestDequeueAllReturnsInReverseOrder) { - TLockFreeStack<int> stack; - - stack.Enqueue(17); - stack.Enqueue(19); - stack.Enqueue(23); - + TLockFreeStack<int> stack; + + stack.Enqueue(17); + stack.Enqueue(19); + stack.Enqueue(23); + TVector<int> r; - - stack.DequeueAll(&r); - - UNIT_ASSERT_VALUES_EQUAL(size_t(3), r.size()); - UNIT_ASSERT_VALUES_EQUAL(23, r.at(0)); - UNIT_ASSERT_VALUES_EQUAL(19, r.at(1)); - UNIT_ASSERT_VALUES_EQUAL(17, r.at(2)); - } - + + stack.DequeueAll(&r); + + UNIT_ASSERT_VALUES_EQUAL(size_t(3), r.size()); + UNIT_ASSERT_VALUES_EQUAL(23, r.at(0)); + UNIT_ASSERT_VALUES_EQUAL(19, r.at(1)); + UNIT_ASSERT_VALUES_EQUAL(17, r.at(2)); + } + Y_UNIT_TEST(TestEnqueueAll) { - TLockFreeStack<int> stack; - + TLockFreeStack<int> stack; + TVector<int> v; TVector<int> expected; - - stack.EnqueueAll(v); // add empty - - v.push_back(2); - v.push_back(3); - v.push_back(5); - expected.insert(expected.end(), v.begin(), v.end()); - stack.EnqueueAll(v); - - v.clear(); - - stack.EnqueueAll(v); // add empty - - v.push_back(7); - v.push_back(11); - v.push_back(13); - v.push_back(17); - expected.insert(expected.end(), v.begin(), v.end()); - stack.EnqueueAll(v); - + + stack.EnqueueAll(v); // add empty + + v.push_back(2); + v.push_back(3); + v.push_back(5); + expected.insert(expected.end(), v.begin(), v.end()); + stack.EnqueueAll(v); + + v.clear(); + + stack.EnqueueAll(v); // add empty + + v.push_back(7); + v.push_back(11); + v.push_back(13); + v.push_back(17); + expected.insert(expected.end(), v.begin(), v.end()); + stack.EnqueueAll(v); + TVector<int> actual; - stack.DequeueAll(&actual); - - UNIT_ASSERT_VALUES_EQUAL(expected.size(), actual.size()); - for (size_t i = 0; i < actual.size(); ++i) { - UNIT_ASSERT_VALUES_EQUAL(expected.at(expected.size() - i - 1), actual.at(i)); - } - } - + stack.DequeueAll(&actual); + + UNIT_ASSERT_VALUES_EQUAL(expected.size(), actual.size()); + for (size_t i = 0; i < actual.size(); ++i) { + UNIT_ASSERT_VALUES_EQUAL(expected.at(expected.size() - i - 1), actual.at(i)); + } + } + Y_UNIT_TEST(CleanInDestructor) { TSimpleSharedPtr<bool> p(new bool); - UNIT_ASSERT_VALUES_EQUAL(1u, p.RefCount()); - - { + UNIT_ASSERT_VALUES_EQUAL(1u, p.RefCount()); + + { TLockFreeStack<TSimpleSharedPtr<bool>> stack; - - stack.Enqueue(p); - stack.Enqueue(p); - - UNIT_ASSERT_VALUES_EQUAL(3u, p.RefCount()); - } - - UNIT_ASSERT_VALUES_EQUAL(1, p.RefCount()); - } + + stack.Enqueue(p); + stack.Enqueue(p); + + UNIT_ASSERT_VALUES_EQUAL(3u, p.RefCount()); + } + + UNIT_ASSERT_VALUES_EQUAL(1, p.RefCount()); + } Y_UNIT_TEST(NoCopyTest) { static unsigned copied = 0; @@ -343,4 +343,4 @@ Y_UNIT_TEST_SUITE(TLockFreeStackTests) { Y_UNIT_TEST(TesMultiThreadMove) { TMultiThreadTester<TMoveTest>().Run(); } -} +} diff --git a/util/thread/pool.cpp b/util/thread/pool.cpp index 25018ea95f..05fad02e9b 100644 --- a/util/thread/pool.cpp +++ b/util/thread/pool.cpp @@ -22,8 +22,8 @@ #include <util/system/condvar.h> #include <util/system/thread.h> -#include <util/datetime/base.h> - +#include <util/datetime/base.h> + #include "factory.h" #include "pool.h" diff --git a/util/thread/pool_ut.cpp b/util/thread/pool_ut.cpp index 86f167f423..893770d0c4 100644 --- a/util/thread/pool_ut.cpp +++ b/util/thread/pool_ut.cpp @@ -1,14 +1,14 @@ #include "pool.h" #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/output.h> #include <util/random/fast.h> #include <util/system/spinlock.h> #include <util/system/thread.h> #include <util/system/mutex.h> #include <util/system/condvar.h> - + struct TThreadPoolTest { TSpinLock Lock; long R = -1; @@ -65,7 +65,7 @@ struct TThreadPoolTest { for (size_t i = 0; i < cnt; ++i) { UNIT_ASSERT(queue->Add(new TTask(this, (long)rand.GenRand()))); - } + } queue->Stop(); @@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(TThreadPoolTest) { UNIT_ASSERT_C(processed, "Not processed"); UNIT_ASSERT_C(destructed, "Not destructed"); } - + Y_UNIT_TEST(TestAddFunc) { TFailAddQueue queue; bool added = queue.AddFunc( diff --git a/util/thread/singleton.h b/util/thread/singleton.h index e9d43bcf20..4a1e05aea0 100644 --- a/util/thread/singleton.h +++ b/util/thread/singleton.h @@ -1,20 +1,20 @@ -#pragma once - -#include <util/system/tls.h> -#include <util/generic/singleton.h> -#include <util/generic/ptr.h> - +#pragma once + +#include <util/system/tls.h> +#include <util/generic/singleton.h> +#include <util/generic/ptr.h> + namespace NPrivate { template <class T, size_t Priority> struct TFastThreadSingletonHelper { static inline T* GetSlow() { return SingletonWithPriority<NTls::TValue<T>, Priority>()->GetPtr(); } - + static inline T* Get() { #if defined(Y_HAVE_FAST_POD_TLS) Y_POD_STATIC_THREAD(T*) fast(nullptr); - + if (Y_UNLIKELY(!fast)) { fast = GetSlow(); } @@ -25,16 +25,16 @@ namespace NPrivate { #endif } }; -} - +} + template <class T, size_t Priority> static inline T* FastTlsSingletonWithPriority() { return ::NPrivate::TFastThreadSingletonHelper<T, Priority>::Get(); -} - -// NB: the singleton is the same for all modules that use -// FastTlsSingleton with the same type parameter. If unique singleton -// required, use unique types. +} + +// NB: the singleton is the same for all modules that use +// FastTlsSingleton with the same type parameter. If unique singleton +// required, use unique types. template <class T> static inline T* FastTlsSingleton() { return FastTlsSingletonWithPriority<T, TSingletonTraits<T>::Priority>(); diff --git a/util/thread/singleton_ut.cpp b/util/thread/singleton_ut.cpp index f02316e6ba..164b1cc184 100644 --- a/util/thread/singleton_ut.cpp +++ b/util/thread/singleton_ut.cpp @@ -1,21 +1,21 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "singleton.h" - -namespace { - struct TFoo { - int i; - TFoo() - : i(0) + +#include "singleton.h" + +namespace { + struct TFoo { + int i; + TFoo() + : i(0) { } - }; -} - + }; +} + Y_UNIT_TEST_SUITE(Tls) { Y_UNIT_TEST(FastThread) { - UNIT_ASSERT_VALUES_EQUAL(0, FastTlsSingleton<TFoo>()->i); - FastTlsSingleton<TFoo>()->i += 3; - UNIT_ASSERT_VALUES_EQUAL(3, FastTlsSingleton<TFoo>()->i); - } -} + UNIT_ASSERT_VALUES_EQUAL(0, FastTlsSingleton<TFoo>()->i); + FastTlsSingleton<TFoo>()->i += 3; + UNIT_ASSERT_VALUES_EQUAL(3, FastTlsSingleton<TFoo>()->i); + } +} diff --git a/util/ya.make b/util/ya.make index d031eb0493..6ebe7e40cf 100644 --- a/util/ya.make +++ b/util/ya.make @@ -10,7 +10,7 @@ NO_UTIL() # stream # string PEERDIR( - util/charset + util/charset contrib/libs/zlib contrib/libs/double-conversion ) @@ -54,8 +54,8 @@ JOIN_SRCS( JOIN_SRCS( all_util.cpp - ysafeptr.cpp - ysaveload.cpp + ysafeptr.cpp + ysaveload.cpp str_stl.cpp ) diff --git a/util/ysaveload.h b/util/ysaveload.h index 966ff3ee85..02efb4049b 100644 --- a/util/ysaveload.h +++ b/util/ysaveload.h @@ -25,12 +25,12 @@ public: }; }; -struct TSerializeException: public yexception { +struct TSerializeException: public yexception { +}; + +struct TLoadEOF: public TSerializeException { }; -struct TLoadEOF: public TSerializeException { -}; - template <class T> static inline void Save(IOutputStream* out, const T& t); diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp index 5be97c467e..76f38bf35c 100644 --- a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp +++ b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp @@ -1400,12 +1400,12 @@ ConvertToMiniKQL(TExprContainer::TPtr expr, promise.SetValue(convRes); } } catch (const TNodeException& ex) { - // TODO: pass backtrace + // TODO: pass backtrace TConvertResult convRes; convRes.Errors.AddIssue(expr->Context.GetPosition(ex.Pos()), ex.what()); promise.SetValue(convRes); } catch (const yexception& ex) { // Catch TProgramBuilder exceptions. - // TODO: pass backtrace + // TODO: pass backtrace TConvertResult convRes; convRes.Errors.AddIssue(NYql::ExceptionToIssue(ex)); promise.SetValue(convRes); diff --git a/ydb/core/client/server/msgbus_server.cpp b/ydb/core/client/server/msgbus_server.cpp index 7ea1444ca9..8a4d8a3bbd 100644 --- a/ydb/core/client/server/msgbus_server.cpp +++ b/ydb/core/client/server/msgbus_server.cpp @@ -595,7 +595,7 @@ void TMessageBusServer::GetTypes(TBusMessageContext &msg) { } void TMessageBusServer::UnknownMessage(TBusMessageContext &msg) { - msg.SendReplyMove(new TBusResponseStatus(MSTATUS_UNKNOWN, "undocumented error 9")); + msg.SendReplyMove(new TBusResponseStatus(MSTATUS_UNKNOWN, "undocumented error 9")); } IActor* TMessageBusServer::CreateProxy() { diff --git a/ydb/core/client/server/msgbus_server_hive_create_tablet.cpp b/ydb/core/client/server/msgbus_server_hive_create_tablet.cpp index e6e0fcabb8..0c41e53f09 100644 --- a/ydb/core/client/server/msgbus_server_hive_create_tablet.cpp +++ b/ydb/core/client/server/msgbus_server_hive_create_tablet.cpp @@ -251,11 +251,11 @@ public: } void Bootstrap(const TActorContext &ctx) { - // handle error from constructor - if (!!ErrorReason) { - return SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, ctx), ctx); - } - + // handle error from constructor + if (!!ErrorReason) { + return SendReplyAndDie(CreateErrorReply(MSTATUS_ERROR, ctx), ctx); + } + NTabletPipe::TClientConfig clientConfig; if (WithRetry) { clientConfig.RetryPolicy = NTabletPipe::TClientRetryPolicy::WithRetries(); diff --git a/ydb/core/client/server/msgbus_server_local_enumerate_tablets.cpp b/ydb/core/client/server/msgbus_server_local_enumerate_tablets.cpp index 4b40d4526b..9f867dddc7 100644 --- a/ydb/core/client/server/msgbus_server_local_enumerate_tablets.cpp +++ b/ydb/core/client/server/msgbus_server_local_enumerate_tablets.cpp @@ -102,7 +102,7 @@ public: void HandleTimeout(const TActorContext &ctx) { Y_UNUSED(ctx); - TAutoPtr<TBusResponse> response(new TBusResponseStatus(MSTATUS_TIMEOUT, "")); + TAutoPtr<TBusResponse> response(new TBusResponseStatus(MSTATUS_TIMEOUT, "")); TBase::SendReplyAndDie(response.Release(), ctx); } diff --git a/ydb/core/client/server/msgbus_server_tracer.cpp b/ydb/core/client/server/msgbus_server_tracer.cpp index ab3c30ffbd..ff93f8d1cb 100644 --- a/ydb/core/client/server/msgbus_server_tracer.cpp +++ b/ydb/core/client/server/msgbus_server_tracer.cpp @@ -173,19 +173,19 @@ public: NBus::TBusMessage* CreateErrorReply(NMsgBusProxy::EResponseStatus status, const TActorContext &ctx) { Y_UNUSED(ctx); - TAutoPtr<NMsgBusProxy::TBusResponse> response(new NMsgBusProxy::TBusResponseStatus(status, "Service not found")); + TAutoPtr<NMsgBusProxy::TBusResponse> response(new NMsgBusProxy::TBusResponseStatus(status, "Service not found")); return response.Release(); } void HandleTimeout(const TActorContext &ctx) { Y_UNUSED(ctx); - TAutoPtr<NMsgBusProxy::TBusResponse> response(new NMsgBusProxy::TBusResponseStatus(NMsgBusProxy::MSTATUS_TIMEOUT, "")); + TAutoPtr<NMsgBusProxy::TBusResponse> response(new NMsgBusProxy::TBusResponseStatus(NMsgBusProxy::MSTATUS_TIMEOUT, "")); this->SendReplyAndDie(response.Release(), ctx); } void HandleUndelivered(TEvents::TEvUndelivered::TPtr& ev, const TActorContext& ctx) { Y_UNUSED(ev); - TAutoPtr<NMsgBusProxy::TBusResponse> response(new NMsgBusProxy::TBusResponseStatus(NMsgBusProxy::MSTATUS_ERROR, "HandleUndelivered")); + TAutoPtr<NMsgBusProxy::TBusResponse> response(new NMsgBusProxy::TBusResponseStatus(NMsgBusProxy::MSTATUS_ERROR, "HandleUndelivered")); response->Record.SetErrorReason("Cannot deliver request to the service"); this->SendReplyAndDie(response.Release(), ctx); } diff --git a/ydb/core/client/server/msgbus_servicereq.h b/ydb/core/client/server/msgbus_servicereq.h index e5a190e94c..704a231dd1 100644 --- a/ydb/core/client/server/msgbus_servicereq.h +++ b/ydb/core/client/server/msgbus_servicereq.h @@ -30,7 +30,7 @@ protected: public: NBus::TBusMessage* CreateErrorReply(EResponseStatus status, const TActorContext &ctx) { Y_UNUSED(ctx); - return new TBusResponseStatus(status, "undocumented error 3"); + return new TBusResponseStatus(status, "undocumented error 3"); } void Bootstrap(const TActorContext &ctx) { diff --git a/ydb/core/driver_lib/cli_utils/cli_scheme_initroot.cpp b/ydb/core/driver_lib/cli_utils/cli_scheme_initroot.cpp index 763fd69fa5..ebca272cc4 100644 --- a/ydb/core/driver_lib/cli_utils/cli_scheme_initroot.cpp +++ b/ydb/core/driver_lib/cli_utils/cli_scheme_initroot.cpp @@ -46,15 +46,15 @@ int SchemeInitRoot(TCommandConfig &cmdConf, int argc, char** argv) { Cout << "status transcript: " << static_cast<NMsgBusProxy::EResponseStatus>(response.GetStatus()) << Endl; if (response.HasTabletId()) Cout << "tabletid: " << response.GetTabletId() << Endl; - - return response.GetStatus() == NMsgBusProxy::MSTATUS_OK ? 0 : 1; + + return response.GetStatus() == NMsgBusProxy::MSTATUS_OK ? 0 : 1; } default: { const char *description = NBus::MessageStatusDescription(status); Cerr << description << Endl; } - return 1; + return 1; } } diff --git a/ydb/core/engine/kikimr_program_builder.cpp b/ydb/core/engine/kikimr_program_builder.cpp index c0d4d3323e..fd32a772ee 100644 --- a/ydb/core/engine/kikimr_program_builder.cpp +++ b/ydb/core/engine/kikimr_program_builder.cpp @@ -402,7 +402,7 @@ TRuntimeNode TKikimrProgramBuilder::EraseRow( TRuntimeNode TKikimrProgramBuilder::Prepare(TRuntimeNode listOfVoid) { auto listType = listOfVoid.GetStaticType(); - AS_TYPE(TListType, listType); + AS_TYPE(TListType, listType); const auto& listDetailedType = static_cast<const TListType&>(*listType); MKQL_ENSURE(listDetailedType.GetItemType()->IsVoid(), "Expected list of void"); @@ -429,7 +429,7 @@ TRuntimeNode TKikimrProgramBuilder::MapParameter( MKQL_ENSURE(callable->GetType()->GetName() == "Parameter", "Expected Parameter callable"); auto listType = list.GetStaticType(); - AS_TYPE(TListType, listType); + AS_TYPE(TListType, listType); const auto& listDetailedType = static_cast<const TListType&>(*listType); auto itemType = listDetailedType.GetItemType(); @@ -455,7 +455,7 @@ TRuntimeNode TKikimrProgramBuilder::FlatMapParameter( MKQL_ENSURE(callable->GetType()->GetName() == "Parameter", "Expected Parameter callable"); auto listType = list.GetStaticType(); - AS_TYPE(TListType, listType); + AS_TYPE(TListType, listType); const auto& listDetailedType = static_cast<const TListType&>(*listType); auto itemType = listDetailedType.GetItemType(); @@ -506,7 +506,7 @@ TRuntimeNode TKikimrProgramBuilder::PartialTake(TRuntimeNode list, TRuntimeNode TRuntimeNode TKikimrProgramBuilder::Bind(TRuntimeNode program, TRuntimeNode parameters, ui32 bindFlags) { auto listType = program.GetStaticType(); - AS_TYPE(TListType, listType); + AS_TYPE(TListType, listType); const auto& listDetailedType = static_cast<const TListType&>(*listType); MKQL_ENSURE(listDetailedType.GetItemType()->IsVoid(), "Expected list of void"); diff --git a/ydb/core/engine/mkql_engine_flat_ut.cpp b/ydb/core/engine/mkql_engine_flat_ut.cpp index 6572d7bd22..e2b12e2d7a 100644 --- a/ydb/core/engine/mkql_engine_flat_ut.cpp +++ b/ydb/core/engine/mkql_engine_flat_ut.cpp @@ -347,7 +347,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLEngineFlatTest) { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &EmptyShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -363,15 +363,15 @@ Y_UNIT_TEST_SUITE(TMiniKQLEngineFlatTest) { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &EmptyShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 2 } @@ -401,7 +401,7 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &EmptyShardResolver), IEngineFlat::EStatus::Aborted); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -423,18 +423,18 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct } } } @@ -480,18 +480,18 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct } } } @@ -540,26 +540,26 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -688,7 +688,7 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -805,7 +805,7 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -847,7 +847,7 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -917,7 +917,7 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -969,7 +969,7 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -1020,7 +1020,7 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -1057,23 +1057,23 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct } } } @@ -1081,7 +1081,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -1217,31 +1217,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -1257,7 +1257,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -1334,31 +1334,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -1374,7 +1374,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -1558,31 +1558,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -1598,7 +1598,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -1679,31 +1679,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -1719,7 +1719,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -1802,31 +1802,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -1842,7 +1842,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -1919,31 +1919,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -1959,7 +1959,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -2153,31 +2153,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -2193,7 +2193,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -2275,31 +2275,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -2315,7 +2315,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -2404,31 +2404,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -2444,7 +2444,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -2526,31 +2526,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -2566,7 +2566,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -2648,31 +2648,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &SingleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -2688,7 +2688,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -2769,26 +2769,26 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &TwoShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes1" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -2806,21 +2806,21 @@ Value { Member { Name: "myRes2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "4" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 2 } @@ -2913,7 +2913,7 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &TwoShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -2981,7 +2981,7 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &TwoShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -3097,7 +3097,7 @@ Value { (simulateFail1 || simulateFail2) ? IEngineFlat::EStatus::Aborted : IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -3175,26 +3175,26 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &NullShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -3256,31 +3256,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &NullShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -3296,7 +3296,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -3348,7 +3348,7 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &NullShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -3393,7 +3393,7 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &NullShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct } )___"; UNIT_ASSERT_STRINGS_EQUAL(resStr, expectedStr); @@ -3453,31 +3453,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &DoubleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -3493,7 +3493,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -3579,31 +3579,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &DoubleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -3619,7 +3619,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -3709,31 +3709,31 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &DoubleShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "List" Type { - Kind: List + Kind: List List { Item { - Kind: Struct + Kind: Struct Struct { Member { Name: "2" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 4608 } @@ -3749,7 +3749,7 @@ Value { Member { Name: "Truncated" Type { - Kind: Data + Kind: Data Data { Scheme: 6 } @@ -4348,15 +4348,15 @@ Value { UNIT_ASSERT_EQUAL(driver.Run(pgm, res, &EmptyShardResolver), IEngineFlat::EStatus::Complete); auto resStr = res.DebugString(); auto expectedStr = R"___(Type { - Kind: Struct + Kind: Struct Struct { Member { Name: "myRes" Type { - Kind: Optional + Kind: Optional Optional { Item { - Kind: Data + Kind: Data Data { Scheme: 2 } diff --git a/ydb/core/engine/mkql_keys.cpp b/ydb/core/engine/mkql_keys.cpp index 984507ec66..05afb89adc 100644 --- a/ydb/core/engine/mkql_keys.cpp +++ b/ydb/core/engine/mkql_keys.cpp @@ -318,7 +318,7 @@ TVector<THolder<TKeyDesc>> ExtractTableKeys(TExploringNodeVisitor& explorer, con } return descList; } - + TTableId ExtractTableId(const TRuntimeNode& node) { if (node.GetStaticType()->IsTuple()) { const TTupleLiteral* tupleNode = AS_VALUE(TTupleLiteral, node); diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 9dd18c0098..d64169d4fc 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -28,7 +28,7 @@ import "ydb/core/protos/node_limits.proto"; import "ydb/core/yq/libs/config/protos/yq_config.proto"; package NKikimrConfig; -option java_package = "ru.yandex.kikimr.proto"; +option java_package = "ru.yandex.kikimr.proto"; message TAffinity { repeated uint32 X = 1; // DEPRECATED: Use `CpuList` instead diff --git a/ydb/core/protos/msgbus.proto b/ydb/core/protos/msgbus.proto index d4d9e4c4c0..df7cda5980 100644 --- a/ydb/core/protos/msgbus.proto +++ b/ydb/core/protos/msgbus.proto @@ -444,7 +444,7 @@ message TSchemeDescribe { }; message TFlatDescribeResponse { - option deprecated = true; + option deprecated = true; optional uint32 Status = 1; optional string Path = 9; diff --git a/ydb/core/protos/msgbus_kv.proto b/ydb/core/protos/msgbus_kv.proto index a588140dae..87c65e5176 100644 --- a/ydb/core/protos/msgbus_kv.proto +++ b/ydb/core/protos/msgbus_kv.proto @@ -1,11 +1,11 @@ package NKikimrClient; -option java_package = "ru.yandex.kikimr.proto"; - - -message TKeyValueRequest { - enum EStorageChannel { - MAIN = 0; - EXTRA = 1; +option java_package = "ru.yandex.kikimr.proto"; + + +message TKeyValueRequest { + enum EStorageChannel { + MAIN = 0; + EXTRA = 1; EXTRA2 = 2; EXTRA3 = 3; EXTRA4 = 4; @@ -16,63 +16,63 @@ message TKeyValueRequest { EXTRA9 = 9; INLINE = 65535; - } - enum EPriority { - REALTIME = 0; - BACKGROUND = 1; - } + } + enum EPriority { + REALTIME = 0; + BACKGROUND = 1; + } enum ETactic { MAX_THROUGHPUT = 0; MIN_LATENCY = 1; } - - message TKeyRange { - optional bytes From = 1; // optional - optional bool IncludeFrom = 2; // default = false - optional bytes To = 3; // optional - optional bool IncludeTo = 4; // default = false - } - - message TCmdDeleteRange { - optional TKeyRange Range = 1; // mandatory - } - message TCmdIncrementGeneration { - } - message TCmdRead { - optional bytes Key = 1; // mandatory - optional uint64 Offset = 2; // (default = 0) - optional uint64 Size = 3; // (default = 0) - full size - optional EPriority Priority = 4; // (default = REALTIME) - } - message TCmdReadRange { - optional TKeyRange Range = 1; // mandatory - optional bool IncludeData = 2; // (default = false) - optional uint64 LimitBytes = 3; // optional, overrides the default one only with a smaller value - optional EPriority Priority = 4; // (default = REALTIME) - } - message TCmdWrite { - optional bytes Key = 1; // mandatory - optional bytes Value = 2; // mandatory - optional EStorageChannel StorageChannel = 3; // (default = MAIN) - optional EPriority Priority = 4; // (default = REALTIME) + + message TKeyRange { + optional bytes From = 1; // optional + optional bool IncludeFrom = 2; // default = false + optional bytes To = 3; // optional + optional bool IncludeTo = 4; // default = false + } + + message TCmdDeleteRange { + optional TKeyRange Range = 1; // mandatory + } + message TCmdIncrementGeneration { + } + message TCmdRead { + optional bytes Key = 1; // mandatory + optional uint64 Offset = 2; // (default = 0) + optional uint64 Size = 3; // (default = 0) - full size + optional EPriority Priority = 4; // (default = REALTIME) + } + message TCmdReadRange { + optional TKeyRange Range = 1; // mandatory + optional bool IncludeData = 2; // (default = false) + optional uint64 LimitBytes = 3; // optional, overrides the default one only with a smaller value + optional EPriority Priority = 4; // (default = REALTIME) + } + message TCmdWrite { + optional bytes Key = 1; // mandatory + optional bytes Value = 2; // mandatory + optional EStorageChannel StorageChannel = 3; // (default = MAIN) + optional EPriority Priority = 4; // (default = REALTIME) optional bytes KeyToCache = 5; // used in PQ optional ETactic Tactic = 6 [default = MAX_THROUGHPUT]; // mandatory, used for non-inline puts only repeated EStorageChannel AutoselectChannel = 7; // when filled, channel is selected automatically from this set - } - message TCmdRename { - optional bytes OldKey = 1; // mandatory - optional bytes NewKey = 2; // mandatory - } - message TCmdCopyRange { - optional TKeyRange Range = 1; // optional; if not set, whole database is copied - optional bytes PrefixToAdd = 2; // mandatory - optional bytes PrefixToRemove = 3; // optional; if set, also acts as filter - } - message TCmdConcat { - repeated bytes InputKeys = 1; // optional - optional bytes OutputKey = 2; // mandatory - optional bool KeepInputs = 3; // optional (default = false) - } + } + message TCmdRename { + optional bytes OldKey = 1; // mandatory + optional bytes NewKey = 2; // mandatory + } + message TCmdCopyRange { + optional TKeyRange Range = 1; // optional; if not set, whole database is copied + optional bytes PrefixToAdd = 2; // mandatory + optional bytes PrefixToRemove = 3; // optional; if set, also acts as filter + } + message TCmdConcat { + repeated bytes InputKeys = 1; // optional + optional bytes OutputKey = 2; // mandatory + optional bool KeepInputs = 3; // optional (default = false) + } message TCmdGetStatus { optional EStorageChannel StorageChannel = 1; // (default = MAIN) } @@ -82,69 +82,69 @@ message TKeyValueRequest { message TCmdSetExecutorFastLogPolicy { optional bool IsAllowed = 1 [default = false]; // mandatory } - - optional uint64 TabletId = 1; // mandatory - optional uint64 Generation = 2; // optional, no generation check is done if missing - optional uint64 Cookie = 3; - repeated TCmdDeleteRange CmdDeleteRange = 4; - optional TCmdIncrementGeneration CmdIncrementGeneration = 5; // conflicts with any other Cmd, must be the only one - repeated TCmdRead CmdRead = 6; - repeated TCmdReadRange CmdReadRange = 7; - repeated TCmdWrite CmdWrite = 8; - repeated TCmdRename CmdRename = 9; - repeated TCmdCopyRange CmdCopyRange = 11; - repeated TCmdConcat CmdConcat = 12; + + optional uint64 TabletId = 1; // mandatory + optional uint64 Generation = 2; // optional, no generation check is done if missing + optional uint64 Cookie = 3; + repeated TCmdDeleteRange CmdDeleteRange = 4; + optional TCmdIncrementGeneration CmdIncrementGeneration = 5; // conflicts with any other Cmd, must be the only one + repeated TCmdRead CmdRead = 6; + repeated TCmdReadRange CmdReadRange = 7; + repeated TCmdWrite CmdWrite = 8; + repeated TCmdRename CmdRename = 9; + repeated TCmdCopyRange CmdCopyRange = 11; + repeated TCmdConcat CmdConcat = 12; repeated TCmdGetStatus CmdGetStatus = 13; optional TCmdTrimLeakedBlobs CmdTrimLeakedBlobs = 14; optional TCmdSetExecutorFastLogPolicy CmdSetExecutorFastLogPolicy = 15; - - optional uint64 DeadlineInstantMs = 10; -} - -// must start from TResponseBase -message TKeyValueResponse { + + optional uint64 DeadlineInstantMs = 10; +} + +// must start from TResponseBase +message TKeyValueResponse { optional uint32 Status = 1; // EResponseStatus from ydb/core/client/base/msgbus.h - optional bytes ProtobufError = 400; - - message TKeyValuePair { - optional bytes Key = 1; - optional bytes Value = 2; - optional uint32 ValueSize = 3; - optional uint64 CreationUnixTime = 4; + optional bytes ProtobufError = 400; + + message TKeyValuePair { + optional bytes Key = 1; + optional bytes Value = 2; + optional uint32 ValueSize = 3; + optional uint64 CreationUnixTime = 4; optional TKeyValueRequest.EStorageChannel StorageChannel = 5; // Returns the _actual_ storage channel optional uint32 Status = 6; // EReplyStatus from ydb/core/protos/base.proto - } - - message TDeleteRangeResult { + } + + message TDeleteRangeResult { optional uint32 Status = 1; // EReplyStatus from ydb/core/protos/base.proto - } - message TIncrementGenerationResult { + } + message TIncrementGenerationResult { optional uint32 Status = 1; // EReplyStatus from ydb/core/protos/base.proto - optional uint64 Generation = 2; - } - message TReadResult { + optional uint64 Generation = 2; + } + message TReadResult { optional uint32 Status = 1; // EReplyStatus from ydb/core/protos/base.proto - optional bytes Value = 2; - optional string Message = 3; - } - message TReadRangeResult { - // may contain partial (truncated) result if Status == OVERRUN + optional bytes Value = 2; + optional string Message = 3; + } + message TReadRangeResult { + // may contain partial (truncated) result if Status == OVERRUN optional uint32 Status = 1; // EReplyStatus from ydb/core/protos/base.proto - repeated TKeyValuePair Pair = 2; - } - message TWriteResult { + repeated TKeyValuePair Pair = 2; + } + message TWriteResult { optional uint32 Status = 1; // EReplyStatus from ydb/core/protos/base.proto optional uint32 StatusFlags = 2; // A set of flags from EStatusFlags ydb/core/protos/blobstorage.proto - } - message TRenameResult { + } + message TRenameResult { optional uint32 Status = 1; // EReplyStatus from ydb/core/protos/base.proto - } - message TCopyRangeResult { + } + message TCopyRangeResult { optional uint32 Status = 1; // EReplyStatus from ydb/core/protos/base.proto - } - message TConcatResult { + } + message TConcatResult { optional uint32 Status = 1; // EReplyStatus from ydb/core/protos/base.proto - } + } message TGetStatusResult { optional uint32 Status = 1; // EReplyStatus from ydb/core/protos/base.proto optional TKeyValueRequest.EStorageChannel StorageChannel = 2; @@ -158,19 +158,19 @@ message TKeyValueResponse { message TSetExecutorFastLogPolicyResult { optional uint32 Status = 1; // EReplyStatus from ydb/core/protos/base.proto } - - optional uint64 Cookie = 2; - repeated TDeleteRangeResult DeleteRangeResult = 3; - optional TIncrementGenerationResult IncrementGenerationResult = 4; - repeated TReadResult ReadResult = 5; - repeated TReadRangeResult ReadRangeResult = 6; - repeated TWriteResult WriteResult = 7; - repeated TRenameResult RenameResult = 8; - repeated TCopyRangeResult CopyRangeResult = 10; - repeated TConcatResult ConcatResult = 11; + + optional uint64 Cookie = 2; + repeated TDeleteRangeResult DeleteRangeResult = 3; + optional TIncrementGenerationResult IncrementGenerationResult = 4; + repeated TReadResult ReadResult = 5; + repeated TReadRangeResult ReadRangeResult = 6; + repeated TWriteResult WriteResult = 7; + repeated TRenameResult RenameResult = 8; + repeated TCopyRangeResult CopyRangeResult = 10; + repeated TConcatResult ConcatResult = 11; repeated TGetStatusResult GetStatusResult = 12; optional TTrimLeakedBlobsResult TrimLeakedBlobsResult = 13; optional TSetExecutorFastLogPolicyResult SetExecutorFastLogPolicyResult = 14; - optional string ErrorReason = 9; // When present contains human-readable error description -} - + optional string ErrorReason = 9; // When present contains human-readable error description +} + diff --git a/ydb/core/protos/ya.make b/ydb/core/protos/ya.make index c0a6fda964..70bb65514c 100644 --- a/ydb/core/protos/ya.make +++ b/ydb/core/protos/ya.make @@ -1,4 +1,4 @@ -PROTO_LIBRARY() +PROTO_LIBRARY() GRPC() @@ -76,7 +76,7 @@ SRCS( minikql_engine.proto msgbus.proto msgbus_health.proto - msgbus_kv.proto + msgbus_kv.proto msgbus_pq.proto netclassifier.proto node_broker.proto diff --git a/ydb/core/scheme/scheme_types_defs.h b/ydb/core/scheme/scheme_types_defs.h index 05ead1a29a..32a3a2b872 100644 --- a/ydb/core/scheme/scheme_types_defs.h +++ b/ydb/core/scheme/scheme_types_defs.h @@ -9,7 +9,7 @@ #define KIKIMR_FOREACH_TYPE(xx, ...) \ - KIKIMR_FOREACH_MINIKQL_TYPE(xx, __VA_ARGS__) \ + KIKIMR_FOREACH_MINIKQL_TYPE(xx, __VA_ARGS__) \ xx(ActorId, TActorId, __VA_ARGS__) \ xx(StepOrderId, TStepOrderId, __VA_ARGS__) \ /**/ diff --git a/ydb/core/scheme_types/scheme_type_registry.h b/ydb/core/scheme_types/scheme_type_registry.h index 3aaace43c3..a82af19267 100644 --- a/ydb/core/scheme_types/scheme_type_registry.h +++ b/ydb/core/scheme_types/scheme_type_registry.h @@ -7,7 +7,7 @@ #include <util/generic/maybe.h> #include <util/generic/singleton.h> #include <util/generic/vector.h> -#include <util/string/builder.h> +#include <util/string/builder.h> namespace NKikimr { @@ -51,13 +51,13 @@ public: } ::TString GetTypeName(TTypeId typeId) const { - if (!typeId) { - return "Null"; - } - auto type = GetType(typeId); + if (!typeId) { + return "Null"; + } + auto type = GetType(typeId); return type.IsKnownType() ? ::TString(type->GetName()) : (TStringBuilder() << "Unknown(" << typeId << ")"); - } - + } + ITypeSP GetKnownType(TTypeId typeId) const { if (!typeId) ythrow yexception() << "Type id must be non zero"; diff --git a/ydb/core/scheme_types/scheme_types_defs.h b/ydb/core/scheme_types/scheme_types_defs.h index a8e0f209f2..bb06b09702 100644 --- a/ydb/core/scheme_types/scheme_types_defs.h +++ b/ydb/core/scheme_types/scheme_types_defs.h @@ -230,18 +230,18 @@ class TDatetime : public IIntegerTypeWithKeyString<ui32, NTypeIds::Datetime, NNa class TTimestamp : public IIntegerTypeWithKeyString<ui64, NTypeIds::Timestamp, NNames::Timestamp> {}; class TInterval : public IIntegerTypeWithKeyString<i64, NTypeIds::Interval, NNames::Interval> {}; -#define KIKIMR_FOREACH_MINIKQL_TYPE_I(name, size, macro, ...) macro(name, T##name, __VA_ARGS__) - -#define KIKIMR_FOREACH_MINIKQL_TYPE(xx, ...) \ - xx(Int32, TInt32, __VA_ARGS__) \ - xx(Uint32, TUint32, __VA_ARGS__) \ - xx(Int64, TInt64, __VA_ARGS__) \ - xx(Uint64, TUint64, __VA_ARGS__) \ +#define KIKIMR_FOREACH_MINIKQL_TYPE_I(name, size, macro, ...) macro(name, T##name, __VA_ARGS__) + +#define KIKIMR_FOREACH_MINIKQL_TYPE(xx, ...) \ + xx(Int32, TInt32, __VA_ARGS__) \ + xx(Uint32, TUint32, __VA_ARGS__) \ + xx(Int64, TInt64, __VA_ARGS__) \ + xx(Uint64, TUint64, __VA_ARGS__) \ xx(Uint8, TUint8, __VA_ARGS__) \ - xx(Bool, TBool, __VA_ARGS__) \ - xx(Double, TDouble, __VA_ARGS__) \ - xx(Float, TFloat, __VA_ARGS__) \ - xx(PairUi64Ui64, TPairUi64Ui64, __VA_ARGS__) \ + xx(Bool, TBool, __VA_ARGS__) \ + xx(Double, TDouble, __VA_ARGS__) \ + xx(Float, TFloat, __VA_ARGS__) \ + xx(PairUi64Ui64, TPairUi64Ui64, __VA_ARGS__) \ xx(String, TString, __VA_ARGS__) \ xx(String4k, TSmallBoundedString, __VA_ARGS__) \ xx(String2m, TLargeBoundedString, __VA_ARGS__) \ @@ -255,23 +255,23 @@ class TInterval : public IIntegerTypeWithKeyString<i64, NTypeIds::Interval, NNam xx(Timestamp, TTimestamp, __VA_ARGS__) \ xx(Interval, TInterval, __VA_ARGS__) \ xx(DyNumber, TDyNumber, __VA_ARGS__) \ - /**/ - - -static inline bool IsValidMinikqlTypeId(TTypeId typeId) { - #define check_valid(v, t, ...) if (NTypeIds::v == typeId) return true; - KIKIMR_FOREACH_MINIKQL_TYPE(check_valid) - #undef check_valid - return false; -} - + /**/ + + +static inline bool IsValidMinikqlTypeId(TTypeId typeId) { + #define check_valid(v, t, ...) if (NTypeIds::v == typeId) return true; + KIKIMR_FOREACH_MINIKQL_TYPE(check_valid) + #undef check_valid + return false; +} + static inline ::TString GetTypeName(TTypeId typeId) { #define getTypeName(v, t, ...) if (NTypeIds::v == typeId) return t::TypeName(); KIKIMR_FOREACH_MINIKQL_TYPE(getTypeName) #undef getTypeName return ::TString("UnknownType(" + ToString(typeId) + ")"); } - + static inline bool TryGetTypeName(TTypeId typeId, ::TString& typeName) { #define getTypeName(v, t, ...) if (NTypeIds::v == typeId) { typeName = t::TypeName(); return true; } KIKIMR_FOREACH_MINIKQL_TYPE(getTypeName) diff --git a/ydb/core/tx/tx_proxy/datareq.cpp b/ydb/core/tx/tx_proxy/datareq.cpp index d3b9b98069..af48b42d60 100644 --- a/ydb/core/tx/tx_proxy/datareq.cpp +++ b/ydb/core/tx/tx_proxy/datareq.cpp @@ -718,10 +718,10 @@ void TDataReq::ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus x->Record.SetStatusCode(code); - for (auto& unresolvedKey : UnresolvedKeys) { - x->Record.AddUnresolvedKeys(unresolvedKey); - } - + for (auto& unresolvedKey : UnresolvedKeys) { + x->Record.AddUnresolvedKeys(unresolvedKey); + } + if (PlanStep) x->Record.SetStep(PlanStep); @@ -1540,7 +1540,7 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr &ev, c if ((ui32)x.Status < (ui32) NSchemeCache::TSchemeCacheRequest::EStatus::OkScheme) { TryToInvalidateTable(x.KeyDescription->TableId, ctx); - TStringStream ss; + TStringStream ss; switch (x.Status) { case NSchemeCache::TSchemeCacheRequest::EStatus::PathErrorNotExist: gotHardResolveError = true; @@ -1556,9 +1556,9 @@ void TDataReq::Handle(TEvTxProxySchemeCache::TEvResolveKeySetResult::TPtr &ev, c } IssueManager.RaiseIssue(MakeIssue(NKikimrIssues::TIssuesIds::GENERIC_RESOLVE_ERROR, ss.Str())); - UnresolvedKeys.push_back(ss.Str()); - } - } + UnresolvedKeys.push_back(ss.Str()); + } + } if (gotHardResolveError) { ReportStatus(TEvTxUserProxy::TEvProposeTransactionStatus::EStatus::ResolveError, NKikimrIssues::TStatusIds::SCHEME_ERROR, true, ctx); diff --git a/ydb/core/tx/tx_proxy/proxy.cpp b/ydb/core/tx/tx_proxy/proxy.cpp index 1dfe1d91e7..55962e7d06 100644 --- a/ydb/core/tx/tx_proxy/proxy.cpp +++ b/ydb/core/tx/tx_proxy/proxy.cpp @@ -8,7 +8,7 @@ TActorId MakeTxProxyID() { } - + TString NKikimr::TEvTxUserProxy::TEvProposeTransactionStatus::ToString() const { TStringStream str; str << "{TEvProposeTransactionStatus"; diff --git a/ydb/library/mkql_proto/protos/minikql.proto b/ydb/library/mkql_proto/protos/minikql.proto index b1dea58a34..2a4243855c 100644 --- a/ydb/library/mkql_proto/protos/minikql.proto +++ b/ydb/library/mkql_proto/protos/minikql.proto @@ -5,7 +5,7 @@ option cc_enable_arenas = true; import "google/protobuf/struct.proto"; enum ETypeKind { - Unknown = 0; + Unknown = 0; Void = 1; Data = 2; Optional = 3; @@ -15,9 +15,9 @@ enum ETypeKind { Dict = 7; Variant = 8; Null = 9; - Reserved_10 = 10; - Reserved_11 = 11; - Reserved_12 = 12; + Reserved_10 = 10; + Reserved_11 = 11; + Reserved_12 = 12; Reserved_13 = 13; Reserved_14 = 14; } @@ -45,7 +45,7 @@ message TTupleType { } message TMember { - required string Name = 1; + required string Name = 1; required TType Type = 2; } @@ -66,16 +66,16 @@ message TDictType { } message TType { - required ETypeKind Kind = 1; - oneof type_type { - TDataType Data = 2; - TOptionalType Optional = 3; - TListType List = 4; - TTupleType Tuple = 5; - TStructType Struct = 6; - TDictType Dict = 7; + required ETypeKind Kind = 1; + oneof type_type { + TDataType Data = 2; + TOptionalType Optional = 3; + TListType List = 4; + TTupleType Tuple = 5; + TStructType Struct = 6; + TDictType Dict = 7; TVariantType Variant = 8; - } + } } message TValuePair { @@ -84,23 +84,23 @@ message TValuePair { } message TValue { - oneof value_value { - bool Bool = 1; - int32 Int32 = 2; - uint32 Uint32 = 3; - int64 Int64 = 4; - uint64 Uint64 = 5; - float Float = 6; - double Double = 7; - bytes Bytes = 8; - string Text = 9; - TValue Optional = 10; + oneof value_value { + bool Bool = 1; + int32 Int32 = 2; + uint32 Uint32 = 3; + int64 Int64 = 4; + uint64 Uint64 = 5; + float Float = 6; + double Double = 7; + bytes Bytes = 8; + string Text = 9; + TValue Optional = 10; fixed64 Low128 = 15; google.protobuf.NullValue NullFlagValue = 18; // Set if current TValue is terminal Null - } - // Logically part of oneof, - // but protobuf does not allow repeated fields in oneof. - // Maybe these fields should have own messages with repeated inside. + } + // Logically part of oneof, + // but protobuf does not allow repeated fields in oneof. + // Maybe these fields should have own messages with repeated inside. repeated TValue List = 11; repeated TValue Tuple = 12; repeated TValue Struct = 13; diff --git a/ydb/library/yql/minikql/mkql_node.cpp b/ydb/library/yql/minikql/mkql_node.cpp index 3641c18ede..4c6f3e45ad 100644 --- a/ydb/library/yql/minikql/mkql_node.cpp +++ b/ydb/library/yql/minikql/mkql_node.cpp @@ -592,12 +592,12 @@ ui32 TStructType::GetMemberIndex(const TStringBuf& name) const { return *index; } - TStringStream ss; - for (ui32 i = 0; i < MembersCount; ++i) { + TStringStream ss; + for (ui32 i = 0; i < MembersCount; ++i) { ss << " " << Members[i].first.Str(); - } + } THROW yexception() << "Member with name '" << name << "' not found; " - << " known members: " << ss.Str() << "."; + << " known members: " << ss.Str() << "."; } TMaybe<ui32> TStructType::FindMemberIndex(const TStringBuf& name) const { diff --git a/ydb/library/yql/minikql/mkql_node_builder.cpp b/ydb/library/yql/minikql/mkql_node_builder.cpp index aeed1cf426..34466b8944 100644 --- a/ydb/library/yql/minikql/mkql_node_builder.cpp +++ b/ydb/library/yql/minikql/mkql_node_builder.cpp @@ -1,5 +1,5 @@ #include "mkql_node_builder.h" -#include "mkql_node_printer.h" +#include "mkql_node_printer.h" #include <util/generic/algorithm.h> @@ -88,7 +88,7 @@ TDataType* UnpackOptionalData(TRuntimeNode data, bool& isOptional) { TDataType* UnpackOptionalData(TType* type, bool& isOptional) { auto unpackedType = UnpackOptional(type, isOptional); - MKQL_ENSURE(unpackedType->IsData(), + MKQL_ENSURE(unpackedType->IsData(), "Expected data or optional of data, actual: " << PrintNode(type, true)); return static_cast<TDataType*>(unpackedType); diff --git a/ydb/library/yql/minikql/mkql_node_printer.cpp b/ydb/library/yql/minikql/mkql_node_printer.cpp index 6802763752..f7515a3005 100644 --- a/ydb/library/yql/minikql/mkql_node_printer.cpp +++ b/ydb/library/yql/minikql/mkql_node_printer.cpp @@ -32,15 +32,15 @@ namespace { void Visit(TTypeType& node) override { Y_UNUSED(node); WriteIndentation(); - Out << "Type (Type)"; - WriteNewline(); + Out << "Type (Type)"; + WriteNewline(); } void Visit(TVoidType& node) override { Y_UNUSED(node); WriteIndentation(); - Out << "Type (Void) "; - WriteNewline(); + Out << "Type (Void) "; + WriteNewline(); } void Visit(TNullType& node) override { @@ -80,20 +80,20 @@ namespace { Out << ", schemeTypeId: "; Out << node.GetSchemeType(); - WriteNewline(); + WriteNewline(); } void Visit(TStructType& node) override { WriteIndentation(); Out << "Type (Struct) with " << node.GetMembersCount() << " members {"; - WriteNewline(); + WriteNewline(); { TIndentScope scope(this); for (ui32 index = 0; index < node.GetMembersCount(); ++index) { WriteIndentation(); - Out << "Member [" << node.GetMemberName(index) << "] : {"; - WriteNewline(); + Out << "Member [" << node.GetMemberName(index) << "] : {"; + WriteNewline(); { TIndentScope scope2(this); @@ -101,26 +101,26 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TListType& node) override { WriteIndentation(); Out << "Type (List) {"; - WriteNewline(); + WriteNewline(); { TIndentScope scope(this); WriteIndentation(); - Out << "List item type: {"; - WriteNewline(); + Out << "List item type: {"; + WriteNewline(); { TIndentScope scope2(this); @@ -128,13 +128,13 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TStreamType& node) override { @@ -222,13 +222,13 @@ namespace { void Visit(TOptionalType& node) override { WriteIndentation(); Out << "Type (Optional) {"; - WriteNewline(); + WriteNewline(); { TIndentScope scope(this); WriteIndentation(); - Out << "Optional item type: {"; - WriteNewline(); + Out << "Optional item type: {"; + WriteNewline(); { TIndentScope scope2(this); @@ -236,25 +236,25 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TDictType& node) override { WriteIndentation(); Out << "Type (Dict) {"; - WriteNewline(); + WriteNewline(); { TIndentScope scope(this); WriteIndentation(); - Out << "Key type: {"; - WriteNewline(); + Out << "Key type: {"; + WriteNewline(); { TIndentScope scope2(this); @@ -262,12 +262,12 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); WriteIndentation(); - Out << "Payload type: {"; - WriteNewline(); + Out << "Payload type: {"; + WriteNewline(); { TIndentScope scope2(this); @@ -275,19 +275,19 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TCallableType& node) override { WriteIndentation(); - Out << "Type (Callable), name: [" << node.GetName() << "] with " << node.GetArgumentsCount() << " args {"; - WriteNewline(); + Out << "Type (Callable), name: [" << node.GetName() << "] with " << node.GetArgumentsCount() << " args {"; + WriteNewline(); { TIndentScope scope(this); @@ -297,8 +297,8 @@ namespace { Out << ", merge disabled"; if (node.GetOptionalArgumentsCount() != 0) Out << ", optional args: " << node.GetOptionalArgumentsCount(); - Out << " : {"; - WriteNewline(); + Out << " : {"; + WriteNewline(); { TIndentScope scope2(this); @@ -306,13 +306,13 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); for (ui32 index = 0; index < node.GetArgumentsCount(); ++index) { WriteIndentation(); const auto& type = node.GetArgumentType(index); Out << "Argument #" << index << " : {"; - WriteNewline(); + WriteNewline(); { TIndentScope scope2(this); @@ -320,14 +320,14 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } if (node.GetPayload()) { WriteIndentation(); - Out << "Payload: {"; - WriteNewline(); + Out << "Payload: {"; + WriteNewline(); { TIndentScope scope2(this); @@ -335,21 +335,21 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TAnyType& node) override { Y_UNUSED(node); WriteIndentation(); Out << "Type (Any) "; - WriteNewline(); + WriteNewline(); } void Visit(TTupleType& node) override { @@ -415,8 +415,8 @@ namespace { void Visit(TVoid& node) override { WriteIndentation(); - Out << "Void {"; - WriteNewline(); + Out << "Void {"; + WriteNewline(); { TIndentScope scope(this); @@ -424,8 +424,8 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TNull& node) override { @@ -475,8 +475,8 @@ namespace { void Visit(TDataLiteral& node) override { WriteIndentation(); - Out << "Data {"; - WriteNewline(); + Out << "Data {"; + WriteNewline(); { TIndentScope scope(this); @@ -484,23 +484,23 @@ namespace { WriteIndentation(); if (node.GetType()->GetSchemeType() == 0) { - Out << "null"; - WriteNewline(); + Out << "null"; + WriteNewline(); } else { Out << TString(node.AsValue().AsStringRef()).Quote(); - WriteNewline(); + WriteNewline(); } } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TStructLiteral& node) override { WriteIndentation(); Out << "Struct {"; - WriteNewline(); + WriteNewline(); { TIndentScope scope(this); @@ -508,8 +508,8 @@ namespace { WriteIndentation(); const auto& value = node.GetValue(index); Out << "Member [" << node.GetType()->GetMemberName(index) << "], " - << (value.IsImmediate() ? "immediate" : "not immediate") << " {"; - WriteNewline(); + << (value.IsImmediate() ? "immediate" : "not immediate") << " {"; + WriteNewline(); { TIndentScope scope2(this); @@ -517,20 +517,20 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TListLiteral& node) override { WriteIndentation(); - Out << "List with " << node.GetItemsCount() << " items {"; - WriteNewline(); + Out << "List with " << node.GetItemsCount() << " items {"; + WriteNewline(); { TIndentScope scope(this); @@ -539,8 +539,8 @@ namespace { for (ui32 i = 0; i < node.GetItemsCount(); ++i) { WriteIndentation(); const auto& item = node.GetItems()[i]; - Out << "Item #" << index << ", " << (item.IsImmediate() ? "immediate" : "not immediate") << " {"; - WriteNewline(); + Out << "Item #" << index << ", " << (item.IsImmediate() ? "immediate" : "not immediate") << " {"; + WriteNewline(); { TIndentScope scope2(this); @@ -548,20 +548,20 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TOptionalLiteral& node) override { WriteIndentation(); - Out << "Optional " << (node.HasItem() ? "with data" : "empty") << " {"; - WriteNewline(); + Out << "Optional " << (node.HasItem() ? "with data" : "empty") << " {"; + WriteNewline(); { TIndentScope scope(this); @@ -569,8 +569,8 @@ namespace { if (node.HasItem()) { WriteIndentation(); const auto& item = node.GetItem(); - Out << "Item " << (item.IsImmediate() ? "immediate" : "not immediate") << " {"; - WriteNewline(); + Out << "Item " << (item.IsImmediate() ? "immediate" : "not immediate") << " {"; + WriteNewline(); { TIndentScope scope2(this); @@ -578,20 +578,20 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TDictLiteral& node) override { WriteIndentation(); - Out << "Dict with " << node.GetItemsCount() << " items {"; - WriteNewline(); + Out << "Dict with " << node.GetItemsCount() << " items {"; + WriteNewline(); { TIndentScope scope(this); @@ -600,7 +600,7 @@ namespace { WriteIndentation(); const auto& item = node.GetItem(index); Out << "Key of item #" << index << ", " << (item.first.IsImmediate() ? "immediate" : "not immediate") << " {"; - WriteNewline(); + WriteNewline(); { TIndentScope scope2(this); @@ -608,12 +608,12 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); WriteIndentation(); Out << "Payload of item #" << index << ", " << (item.second.IsImmediate() ? "immediate" : "not immediate") << " {"; - WriteNewline(); + WriteNewline(); { TIndentScope scope2(this); @@ -621,14 +621,14 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TCallable& node) override { @@ -636,8 +636,8 @@ namespace { Out << "Callable"; if (node.GetUniqueId() != 0) Out << ", uniqueId: " << node.GetUniqueId(); - Out << " {"; - WriteNewline(); + Out << " {"; + WriteNewline(); { TIndentScope scope(this); @@ -646,8 +646,8 @@ namespace { for (ui32 index = 0; index < node.GetInputsCount(); ++index) { WriteIndentation(); const auto& input = node.GetInput(index); - Out << "Input #" << index << ", " << (input.IsImmediate() ? "immediate" : "not immediate") << " {"; - WriteNewline(); + Out << "Input #" << index << ", " << (input.IsImmediate() ? "immediate" : "not immediate") << " {"; + WriteNewline(); { TIndentScope scope3(this); @@ -655,14 +655,14 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } if (node.HasResult()) { WriteIndentation(); - Out << "Result, " << (node.GetResult().IsImmediate() ? "immediate" : "not immediate") << " {"; - WriteNewline(); + Out << "Result, " << (node.GetResult().IsImmediate() ? "immediate" : "not immediate") << " {"; + WriteNewline(); { TIndentScope scope3(this); @@ -670,20 +670,20 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TAny& node) override { WriteIndentation(); Out << "Any " << (node.HasItem() ? "with data" : "empty") << " {"; - WriteNewline(); + WriteNewline(); { TIndentScope scope(this); @@ -706,8 +706,8 @@ namespace { } WriteIndentation(); - Out << "}"; - WriteNewline(); + Out << "}"; + WriteNewline(); } void Visit(TTupleLiteral& node) override { @@ -799,24 +799,24 @@ namespace { private: void WriteIndentation() { - if (SingleLine) { - } else { - for (ui32 i = 0; i < 2 * Indent; ++i) { - Out << ' '; - } - } - } - - void WriteNewline() { - if (SingleLine) { + if (SingleLine) { + } else { + for (ui32 i = 0; i < 2 * Indent; ++i) { + Out << ' '; + } + } + } + + void WriteNewline() { + if (SingleLine) { Out << ' '; - } else { - Out << '\n'; + } else { + Out << '\n'; } } private: - const bool SingleLine; + const bool SingleLine; TStringStream Out; ui32 Indent; }; @@ -824,18 +824,18 @@ namespace { TString PrintNode(const TNode* node, bool singleLine) { TPrintVisitor visitor(singleLine); - const_cast<TNode*>(node)->Accept(visitor); + const_cast<TNode*>(node)->Accept(visitor); return visitor.ToString(); } } } - -template <> -void Out<NKikimr::NMiniKQL::TType>( + +template <> +void Out<NKikimr::NMiniKQL::TType>( IOutputStream& os, - TTypeTraits<NKikimr::NMiniKQL::TType>::TFuncParam t -) -{ + TTypeTraits<NKikimr::NMiniKQL::TType>::TFuncParam t +) +{ os << PrintNode(&t, true); -} +} diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp index e2de109fac..47ae6e4c8f 100644 --- a/ydb/library/yql/minikql/mkql_program_builder.cpp +++ b/ydb/library/yql/minikql/mkql_program_builder.cpp @@ -353,7 +353,7 @@ TRuntimeNode TProgramBuilder::Member(TRuntimeNode structObj, const std::string_v } TCallableBuilder callableBuilder(Env, __func__, memberType); - callableBuilder.Add(structObj); + callableBuilder.Add(structObj); callableBuilder.Add(NewDataLiteral<ui32>(memberIndex)); return TRuntimeNode(callableBuilder.Build(), false); } @@ -434,7 +434,7 @@ TRuntimeNode TProgramBuilder::Zip(const TArrayRef<const TRuntimeNode>& lists) { continue; } - AS_TYPE(TListType, list.GetStaticType()); + AS_TYPE(TListType, list.GetStaticType()); auto itemType = static_cast<const TListType&>(*list.GetStaticType()).GetItemType(); tupleTypes.push_back(itemType); } @@ -461,7 +461,7 @@ TRuntimeNode TProgramBuilder::ZipAll(const TArrayRef<const TRuntimeNode>& lists) continue; } - AS_TYPE(TListType, list.GetStaticType()); + AS_TYPE(TListType, list.GetStaticType()); auto itemType = static_cast<const TListType&>(*list.GetStaticType()).GetItemType(); tupleTypes.push_back(TOptionalType::Create(itemType, Env)); } @@ -1788,7 +1788,7 @@ TRuntimeNode TProgramBuilder::SqueezeToList(TRuntimeNode flow, TRuntimeNode limi TRuntimeNode TProgramBuilder::Append(TRuntimeNode list, TRuntimeNode item) { auto listType = list.GetStaticType(); - AS_TYPE(TListType, listType); + AS_TYPE(TListType, listType); const auto& listDetailedType = static_cast<const TListType&>(*listType); auto itemType = item.GetStaticType(); @@ -1802,7 +1802,7 @@ TRuntimeNode TProgramBuilder::Append(TRuntimeNode list, TRuntimeNode item) { TRuntimeNode TProgramBuilder::Prepend(TRuntimeNode item, TRuntimeNode list) { auto listType = list.GetStaticType(); - AS_TYPE(TListType, listType); + AS_TYPE(TListType, listType); const auto& listDetailedType = static_cast<const TListType&>(*listType); auto itemType = item.GetStaticType(); diff --git a/ydb/library/yql/protos/ya.make b/ydb/library/yql/protos/ya.make index 82c5281ef6..c9200505fb 100644 --- a/ydb/library/yql/protos/ya.make +++ b/ydb/library/yql/protos/ya.make @@ -1,4 +1,4 @@ -PROTO_LIBRARY() +PROTO_LIBRARY() OWNER(g:yql) diff --git a/ydb/public/lib/base/msgbus.h b/ydb/public/lib/base/msgbus.h index f02762b577..f11bd9771f 100644 --- a/ydb/public/lib/base/msgbus.h +++ b/ydb/public/lib/base/msgbus.h @@ -47,7 +47,7 @@ enum { MTYPE_CLIENT_FLAT_TX_REQUEST = 10432, MTYPE_CLIENT_FLAT_TX_STATUS_REQUEST = 10434, MTYPE_CLIENT_OLD_FLAT_DESCRIBE_REQUEST = 10435, // deprecated - MTYPE_CLIENT_OLD_FLAT_DESCRIBE_RESPONSE = 10436, // deprecated + MTYPE_CLIENT_OLD_FLAT_DESCRIBE_RESPONSE = 10436, // deprecated MTYPE_CLIENT_CREATE_TABLET = 10437, MTYPE_CLIENT_LOAD_REQUEST = 10438, MTYPE_CLIENT_LOAD_RESPONSE = 10439, @@ -157,9 +157,9 @@ public: TBusResponseStatus(EResponseStatus status, const TString& text = TString()) { Record.SetStatus(status); - if (text) { + if (text) { Record.SetErrorReason(text); - } + } } }; @@ -201,9 +201,9 @@ public: RegisterType(new TBusTypesResponse); RegisterType(new TBusMessageBusTraceRequest); RegisterType(new TBusMessageBusTraceStatus); - RegisterType(new TBusHiveCreateTablet); + RegisterType(new TBusHiveCreateTablet); RegisterType(new TBusOldHiveCreateTablet); - RegisterType(new TBusHiveCreateTabletResult); + RegisterType(new TBusHiveCreateTabletResult); RegisterType(new TBusLocalEnumerateTablets); RegisterType(new TBusOldLocalEnumerateTablets); RegisterType(new TBusLocalEnumerateTabletsResult); @@ -220,7 +220,7 @@ public: RegisterType(new TBusSchemeOperationStatus); RegisterType(new TBusSchemeDescribe); RegisterType(new TBusOldFlatDescribeRequest); - RegisterType(new TBusOldFlatDescribeResponse); + RegisterType(new TBusOldFlatDescribeResponse); RegisterType(new TBusBsTestLoadRequest); RegisterType(new TBusBsTestLoadResponse); RegisterType(new TBusBsGetRequest); |