YT-19688: Move mrjob to ydb/library/yql/tools

13 files changed, 285 insertions, 0 deletions

diff --git a/ydb/library/yql/CMakeLists.txt b/ydb/library/yql/CMakeLists.txt
index 4fc40ccc667..7f41afad8ef 100644
--- a/ydb/library/yql/CMakeLists.txt
+++ b/ydb/library/yql/CMakeLists.txt
@@ -15,5 +15,6 @@ add_subdirectory(protos)
 add_subdirectory(providers)
 add_subdirectory(public)
 add_subdirectory(sql)
+add_subdirectory(tools)
 add_subdirectory(udfs)
 add_subdirectory(utils)
diff --git a/ydb/library/yql/tools/CMakeLists.txt b/ydb/library/yql/tools/CMakeLists.txt
new file mode 100644
index 00000000000..4c2a3daa2ea
--- /dev/null
+++ b/ydb/library/yql/tools/CMakeLists.txt
@@ -0,0 +1,9 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(mrjob)
diff --git a/ydb/library/yql/tools/mrjob/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/tools/mrjob/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..247502db46a
--- /dev/null
+++ b/ydb/library/yql/tools/mrjob/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,39 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(mrjob)
+target_compile_options(mrjob PRIVATE
+  -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(mrjob PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-cpuid_check
+  cpp-mapreduce-client
+  udf-service-terminate_policy
+  providers-common-gateway
+  yql-utils-backtrace
+  yql-parser-pg_wrapper
+  yql-sql-pg
+  providers-yt-job
+)
+target_link_options(mrjob PRIVATE
+  -Wl,-platform_version,macos,11.0,11.0
+  -fPIC
+  -fPIC
+  -framework
+  CoreFoundation
+)
+target_sources(mrjob PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/mrjob/mrjob.cpp
+)
+target_allocator(mrjob
+  cpp-malloc-jemalloc
+)
+vcs_info(mrjob)
diff --git a/ydb/library/yql/tools/mrjob/CMakeLists.linux-aarch64.txt b/ydb/library/yql/tools/mrjob/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..af1caa79edd
--- /dev/null
+++ b/ydb/library/yql/tools/mrjob/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,43 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(mrjob)
+target_compile_options(mrjob PRIVATE
+  -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(mrjob PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  cpp-mapreduce-client
+  udf-service-terminate_policy
+  providers-common-gateway
+  yql-utils-backtrace
+  yql-parser-pg_wrapper
+  yql-sql-pg
+  providers-yt-job
+)
+target_link_options(mrjob PRIVATE
+  -ldl
+  -lrt
+  -Wl,--no-as-needed
+  -fPIC
+  -fPIC
+  -lpthread
+  -lrt
+  -ldl
+  -lutil
+)
+target_sources(mrjob PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/mrjob/mrjob.cpp
+)
+target_allocator(mrjob
+  cpp-malloc-jemalloc
+)
+vcs_info(mrjob)
diff --git a/ydb/library/yql/tools/mrjob/CMakeLists.linux-x86_64.txt b/ydb/library/yql/tools/mrjob/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..18e74ea77df
--- /dev/null
+++ b/ydb/library/yql/tools/mrjob/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,44 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(mrjob)
+target_compile_options(mrjob PRIVATE
+  -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(mrjob PUBLIC
+  contrib-libs-linux-headers
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-cpuid_check
+  cpp-mapreduce-client
+  udf-service-terminate_policy
+  providers-common-gateway
+  yql-utils-backtrace
+  yql-parser-pg_wrapper
+  yql-sql-pg
+  providers-yt-job
+)
+target_link_options(mrjob PRIVATE
+  -ldl
+  -lrt
+  -Wl,--no-as-needed
+  -fPIC
+  -fPIC
+  -lpthread
+  -lrt
+  -ldl
+  -lutil
+)
+target_sources(mrjob PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/mrjob/mrjob.cpp
+)
+target_allocator(mrjob
+  cpp-malloc-jemalloc
+)
+vcs_info(mrjob)
diff --git a/ydb/library/yql/tools/mrjob/CMakeLists.txt b/ydb/library/yql/tools/mrjob/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/tools/mrjob/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+  include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+  include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+  include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/tools/mrjob/CMakeLists.windows-x86_64.txt b/ydb/library/yql/tools/mrjob/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..a2c958a6d01
--- /dev/null
+++ b/ydb/library/yql/tools/mrjob/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,32 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(mrjob)
+target_compile_options(mrjob PRIVATE
+  -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(mrjob PUBLIC
+  contrib-libs-cxxsupp
+  yutil
+  library-cpp-cpuid_check
+  cpp-mapreduce-client
+  udf-service-terminate_policy
+  providers-common-gateway
+  yql-utils-backtrace
+  yql-parser-pg_wrapper
+  yql-sql-pg
+  providers-yt-job
+)
+target_sources(mrjob PRIVATE
+  ${CMAKE_SOURCE_DIR}/ydb/library/yql/tools/mrjob/mrjob.cpp
+)
+target_allocator(mrjob
+  system_allocator
+)
+vcs_info(mrjob)
diff --git a/ydb/library/yql/tools/mrjob/mrjob.cpp b/ydb/library/yql/tools/mrjob/mrjob.cpp
new file mode 100644
index 00000000000..5cdd262eea0
--- /dev/null
+++ b/ydb/library/yql/tools/mrjob/mrjob.cpp
@@ -0,0 +1,29 @@
+#include <ydb/library/yql/providers/yt/job/yql_job_registry.h>
+
+#include <ydb/library/yql/utils/backtrace/backtrace.h>
+
+#include <yt/cpp/mapreduce/client/init.h>
+
+#include <util/system/yassert.h>
+#include <util/system/mlock.h>
+
+int main(int argc, const char *argv[]) {
+    Y_UNUSED(NKikimr::NUdf::GetStaticSymbols());
+    try {
+        LockAllMemory(LockCurrentMemory | LockFutureMemory);
+    } catch (yexception&) {
+        Cerr << "mlockall failed, but that's fine" << Endl;
+    }
+
+    NYql::NBacktrace::RegisterKikimrFatalActions();
+    NYql::NBacktrace::EnableKikimrSymbolize();
+
+    try {
+        NYT::Initialize(argc, argv);
+    } catch (...) {
+        Cerr << CurrentExceptionMessage();
+        return -1;
+    }
+
+    Y_FAIL("This binary should not be called directly");
+}
diff --git a/ydb/library/yql/tools/mrjob/test/test.py b/ydb/library/yql/tools/mrjob/test/test.py
new file mode 100644
index 00000000000..c924e4ba442
--- /dev/null
+++ b/ydb/library/yql/tools/mrjob/test/test.py
@@ -0,0 +1,29 @@
+import os
+import os.path
+import re
+import subprocess
+
+import yatest.common
+
+
+def test_libc():
+    mrjob_dir = yatest.common.binary_path('ydb/library/yql/tools/mrjob')
+    mrjob_path = os.path.join(mrjob_dir, 'mrjob')
+    tools_path = os.path.dirname(yatest.common.cxx_compiler_path())
+    nm_path = os.path.join(tools_path, 'llvm-nm')
+    readelf_path = os.path.join(tools_path, 'readelf')
+    if os.path.isfile(nm_path):
+        result = subprocess.check_output([nm_path, mrjob_path])
+    elif os.path.isfile(readelf_path):
+        result = subprocess.check_output([readelf_path, '-a', mrjob_path])
+    else:
+        assert False, 'neither llvm-nm nor readelf found, checked paths: %s' % str((readelf_path, nm_path))
+
+    glibc_tag_count = 0
+    for line in result.decode().split('\n'):
+        glibc_tag = re.search('GLIBC_[0-9\.]+', line)
+        if glibc_tag:
+            glibc_tag_count += 1
+            parts = glibc_tag.group().split('.')
+            assert len(parts) > 1
+            assert int(parts[1]) <= 11
diff --git a/ydb/library/yql/tools/mrjob/test/ya.make b/ydb/library/yql/tools/mrjob/test/ya.make
new file mode 100644
index 00000000000..d1dca051d2e
--- /dev/null
+++ b/ydb/library/yql/tools/mrjob/test/ya.make
@@ -0,0 +1,10 @@
+IF(OS_LINUX)
+    PY3TEST()
+    TEST_SRCS(test.py)
+
+    DEPENDS(
+        ydb/library/yql/tools/mrjob
+    )
+
+    END()
+ENDIF()
diff --git a/ydb/library/yql/tools/mrjob/ya.make b/ydb/library/yql/tools/mrjob/ya.make
new file mode 100644
index 00000000000..142414ac1a1
--- /dev/null
+++ b/ydb/library/yql/tools/mrjob/ya.make
@@ -0,0 +1,27 @@
+PROGRAM(mrjob)
+
+ALLOCATOR(J)
+
+SRCS(
+    mrjob.cpp
+)
+
+IF (OS_LINUX)
+    # prevent external python extensions to lookup protobuf symbols (and maybe
+    # other common stuff) in main binary
+    EXPORTS_SCRIPT(${ARCADIA_ROOT}/yql/tools/exports.symlist)
+ENDIF()
+
+PEERDIR(
+    yt/cpp/mapreduce/client
+    ydb/library/yql/public/udf/service/terminate_policy
+    ydb/library/yql/providers/common/gateway
+    ydb/library/yql/utils/backtrace
+    ydb/library/yql/parser/pg_wrapper
+    ydb/library/yql/sql/pg
+    ydb/library/yql/providers/yt/job
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/tools/ya.make b/ydb/library/yql/tools/ya.make
new file mode 100644
index 00000000000..abaf4d8a13a
--- /dev/null
+++ b/ydb/library/yql/tools/ya.make
@@ -0,0 +1,4 @@
+RECURSE(
+    mrjob
+    mrjob/test
+)
diff --git a/ydb/library/yql/ya.make b/ydb/library/yql/ya.make
index 94a30b6b957..83b1b9e9270 100644
--- a/ydb/library/yql/ya.make
+++ b/ydb/library/yql/ya.make
@@ -23,6 +23,7 @@ RECURSE(
     providers
     public
     sql
+    tools
     udfs
     utils
 )