KIKIMR-11746: fix potential problem with INT_MIN value in memcmp() result

ref:ef4f11697df6c1118217293b23140d5b3e246cf6

1 files changed, 5 insertions, 3 deletions

diff --git a/ydb/core/formats/replace_key.h b/ydb/core/formats/replace_key.h
index fa4fb094955..c902ea46a67 100644
--- a/ydb/core/formats/replace_key.h
+++ b/ydb/core/formats/replace_key.h
@@ -286,10 +286,12 @@ private:
         if constexpr (std::is_same_v<T, arrow::util::string_view>) {
             size_t minSize = (x.size() < y.size()) ? x.size() : y.size();
             int cmp = memcmp(x.data(), y.data(), minSize);
-            if (!cmp) {
-                return CompareValueNotNull(x.size(), y.size());
+            if (cmp < 0) {
+                return -1; // avoid INT_MIN as negative cmp. We require "-negative is positive" for result.
+            } else if (cmp > 0) {
+                return 1;
             }
-            return cmp;
+            return CompareValueNotNull(x.size(), y.size());
         } else {
             if (x < y) {
                 return -1;