move some scripts to build/internal/scrips

5 files changed, 0 insertions, 468 deletions

diff --git a/build/scripts/build_mn.py b/build/scripts/build_mn.py
deleted file mode 100755
index 5bb03c247c..0000000000
--- a/build/scripts/build_mn.py
+++ /dev/null
@@ -1,330 +0,0 @@
-#!/usr/bin/env python
-# Ymake MatrixNet support
-
-import sys
-import os
-import shutil
-import re
-import subprocess
-
-
-def get_value(val):
-    dct = val.split('=', 1)
-    if len(dct) > 1:
-        return dct[1]
-    return ''
-
-
-class BuildMnBase(object):
-    def Run(self, mninfo, mnname, mnrankingSuffix, mncppPath, check=False, ptr=False, multi=False):
-        self.mninfo = mninfo
-        self.mnname = mnname
-        self.mnrankingSuffix = mnrankingSuffix
-        self.mncppPath = mncppPath
-        self.check = check
-        self.ptr = ptr
-        self.multi = multi
-        dataprefix = "MN_External_"
-        mninfoName = os.path.basename(self.mninfo)
-        data = dataprefix + mnname
-        datasize = data + "Size"
-
-        if self.multi:
-            if self.ptr:
-                mntype = "const NMatrixnet::TMnMultiCategPtr"
-                mnload = "(new NMatrixnet::TMnMultiCateg( {1}, {2}, \"{0}\"))".format(mninfoName, data, datasize)
-            else:
-                mntype = "const NMatrixnet::TMnMultiCateg"
-                mnload = "({1}, {2}, \"{0}\")".format(mninfoName, data, datasize)
-        else:
-            if self.ptr:
-                mntype = "const NMatrixnet::TMnSsePtr"
-                mnload = "(new NMatrixnet::TMnSseInfo({1}, {2}, \"{0}\"))".format(mninfoName, data, datasize)
-            else:
-                mntype = "const NMatrixnet::TMnSseInfo"
-                mnload = "({1}, {2}, \"{0}\")".format(mninfoName, data, datasize)
-
-        if self.check:
-            self.CheckMn()
-
-        mncpptmpPath = self.mncppPath + ".tmp"
-        mncpptmp = open(mncpptmpPath, 'w')
-
-        if self.multi:
-            mncpptmp.write("#include <kernel/matrixnet/mn_multi_categ.h>\n")
-        else:
-            mncpptmp.write("#include <kernel/matrixnet/mn_sse.h>\n")
-
-        rodatapath = os.path.dirname(self.mncppPath) + "/" + dataprefix + self.mnname + ".rodata"
-        mncpptmp.write("namespace{\n")
-        mncpptmp.write("    extern \"C\" {\n")
-        mncpptmp.write("        extern const unsigned char {1}{0}[];\n".format(self.mnname, dataprefix))
-        mncpptmp.write("        extern const ui32 {1}{0}Size;\n".format(self.mnname, dataprefix))
-        mncpptmp.write("    }\n")
-        mncpptmp.write("}\n")
-        archiverCall = subprocess.Popen([self.archiver, "-q", "-p", "-o", rodatapath, self.mninfo], stdout=None, stderr=subprocess.PIPE)
-        archiverCall.wait()
-        mncpptmp.write("extern {0} {1};\n".format(mntype, self.mnname))
-        mncpptmp.write("{0} {1}{2};".format(mntype, self.mnname, mnload))
-        mncpptmp.close()
-        shutil.move(mncpptmpPath, self.mncppPath)
-
-    def CheckMn(self):
-        if not self.fml_unused_tool:
-            print >>sys.stderr, "fml_unused_tool undefined!"
-        failed_msg = "fml_unused_tool failed: {0} -A {1} -e -r {2}".format(self.fml_unused_tool, self.SrcRoot, self.mninfo)
-        assert not subprocess.call([self.fml_unused_tool, "-A", self.SrcRoot, "-e", "-r", self.mninfo]), failed_msg
-
-
-class BuildMn(BuildMnBase):
-    def Run(self, argv):
-        if len(argv) < 6:
-            print >>sys.stderr, "BuildMn.Run(<ARCADIA_ROOT> <archiver> <mninfo> <mnname> <mnrankingSuffix> <cppOutput> [params...])"
-            sys.exit(1)
-
-        self.SrcRoot = argv[0]
-        self.archiver = argv[1]
-
-        mninfo = argv[2]
-        mnname = argv[3]
-        mnrankingSuffix = argv[4]
-        mncppPath = argv[5]
-        check = False
-        ptr = False
-        multi = False
-        self.fml_unused_tool = ''
-        for param in argv[6:]:
-            if param == "CHECK":
-                check = True
-            elif param == "PTR":
-                ptr = True
-            elif param == "MULTI":
-                multi = True
-            elif param.startswith('fml_tool='):
-                self.fml_unused_tool = get_value(param)
-            else:
-                print >>sys.stdout, "Unknown param: {0}".format(param)
-        super(BuildMn, self).Run(mninfo, mnname, mnrankingSuffix, mncppPath, check=check, ptr=ptr, multi=multi)
-
-
-class BuildMns(BuildMnBase):
-    def InitBase(self, listname, mnrankingSuffix):
-        self.autogen = '// DO NOT EDIT THIS FILE DIRECTLY, AUTOGENERATED!\n'
-        self.mnrankingSuffix = mnrankingSuffix
-        self.mnlistname = listname + mnrankingSuffix
-        self.mnlistelem = "const NMatrixnet::TMnSsePtr*"
-        mnlisttype = "TMap< TString, {0} >".format(self.mnlistelem)
-        self.mnlist = "const {0} {1}".format(mnlisttype, self.mnlistname)
-
-        self.mnmultilistname = "{0}{1}Multi".format(listname, self.mnrankingSuffix)
-        self.mnmultilistelem = "const NMatrixnet::TMnMultiCategPtr*"
-        mnmultilisttype = "TMap< TString, {0} >".format(self.mnmultilistelem)
-        self.mnmultilist = "const {0} {1}".format(mnmultilisttype, self.mnmultilistname)
-
-    def InitForAll(self, argv):
-        if len(argv) < 8:
-            print >>sys.stderr, "BuildMns.InitForAll(<ARCADIA_ROOT> <BINDIR> <archiver>  <listname> <mnranking_suffix> <hdrfile> <srcfile> <mninfos> [fml_tool=<fml_unused_tool> CHECK])"
-            sys.exit(1)
-
-        bmns_args = []
-        self.check = False
-        self.fml_unused_tool = ''
-        for arg in argv:
-            if arg == "CHECK":
-                self.check = True
-            elif arg.startswith('fml_tool='):
-                self.fml_unused_tool = get_value(arg)
-            else:
-                bmns_args.append(arg)
-
-        self.SrcRoot = bmns_args[0]
-        self.BINDIR = bmns_args[1]
-        self.archiver = bmns_args[2]
-        self.listname = bmns_args[3]
-        self.mnrankingSuffix = get_value(bmns_args[4])
-        self.hdrfile = bmns_args[5]
-        self.srcfile = bmns_args[6]
-        self.mninfos = bmns_args[7:]
-
-        self.InitBase(self.listname, self.mnrankingSuffix)
-
-    def InitForHeader(self, argv):
-        if len(argv) < 4:
-            print >>sys.stderr, "BuildMns.InitForHeader(<listname> <rankingSuffix> <hdrfile> <mninfos...>)"
-            sys.exit(1)
-
-        self.listname = argv[0]
-        self.mnrankingSuffix = get_value(argv[1])
-        self.hdrfile = argv[2]
-        self.mninfos = argv[3:]
-
-        self.InitBase(self.listname, self.mnrankingSuffix)
-
-    def InitForCpp(self, argv):
-        if len(argv) < 5:
-            print >>sys.stderr, "BuildMns.InitForCpp(<listname> <rankingSuffix> <hdrfile> <srcfile> <mninfos...>)"
-            sys.exit(1)
-
-        self.listname = argv[0]
-        self.mnrankingSuffix = get_value(argv[1])
-        self.hdrfile = argv[2]
-        self.srcfile = argv[3]
-        self.mninfos = argv[4:]
-
-        self.InitBase(self.listname, self.mnrankingSuffix)
-
-    def InitForFiles(self, argv):
-        if len(argv) < 7:
-            print >>sys.stderr, "BuildMns.InitForFiles(<ARCADIA_ROOT> <BINDIR> <archiver> <fml_unused_tool> <listname> <rankingSuffix> <mninfos...> [CHECK])"
-            sys.exit(1)
-
-        bmns_args = []
-        self.check = False
-        self.fml_unused_tool = ''
-        for arg in argv:
-            if arg == "CHECK":
-                self.check = True
-            elif arg.startswith('fml_tool='):
-                self.fml_unused_tool = get_value(arg)
-            else:
-                bmns_args.append(arg)
-
-        self.SrcRoot = bmns_args[0]
-        self.BINDIR = bmns_args[1]
-        self.archiver = bmns_args[2]
-        self.listname = bmns_args[3]
-        self.mnrankingSuffix = get_value(bmns_args[4])
-        self.mninfos = bmns_args[5:]
-
-    def BuildMnsHeader(self):
-        if self.mninfos:
-            self.mninfos = sorted(set(self.mninfos))
-
-        tmpHdrPath = self.hdrfile + ".tmp"
-        tmpHdrFile = open(tmpHdrPath, 'w')
-
-        tmpHdrFile.write(self.autogen)
-        tmpHdrFile.write("#include <kernel/matrixnet/mn_sse.h>\n")
-        tmpHdrFile.write("#include <kernel/matrixnet/mn_multi_categ.h>\n\n")
-        tmpHdrFile.write("extern {0};\n".format(self.mnlist))
-        tmpHdrFile.write("extern {0};\n".format(self.mnmultilist))
-
-        for item in self.mninfos:
-            mnfilename = os.path.basename(item)
-            mnfilename, ext = os.path.splitext(mnfilename)
-
-            mnname = re.sub("[^-a-zA-Z0-9_]", "_", mnfilename)
-
-            if ext == ".info":
-                mnname = "staticMn{0}{1}Ptr".format(self.mnrankingSuffix, mnname)
-                tmpHdrFile.write("extern const NMatrixnet::TMnSsePtr {0};\n".format(mnname))
-            elif ext == ".mnmc":
-                mnname = "staticMnMulti{0}{1}Ptr".format(self.mnrankingSuffix, mnname)
-                tmpHdrFile.write("extern const NMatrixnet::TMnMultiCategPtr {0};\n".format(mnname))
-
-        tmpHdrFile.close()
-        shutil.move(tmpHdrPath, self.hdrfile)
-
-    def BuildMnFiles(self):
-        for item in self.mninfos:
-            mnfilename = os.path.basename(item)
-            mnfilename, ext = os.path.splitext(mnfilename)
-
-            mnname = re.sub("[^-a-zA-Z0-9_]", "_", mnfilename)
-
-            if ext == ".info":
-                mnname = "staticMn{0}{1}Ptr".format(self.mnrankingSuffix, mnname)
-                super(BuildMns, self).Run(item, mnname, self.mnrankingSuffix, self.BINDIR + "/mn.{0}.cpp".format(mnname), check=self.check, ptr=True, multi=False)
-            elif ext == ".mnmc":
-                mnname = "staticMnMulti{0}{1}Ptr".format(self.mnrankingSuffix, mnname)
-                # BUILD_MN_PTR_MULTI
-                super(BuildMns, self).Run(item, mnname, self.mnrankingSuffix, self.BINDIR + "/mnmulti.{0}.cpp".format(mnname), check=False, ptr=True, multi=True)
-
-    def BuildMnsCpp(self):
-        if self.mninfos:
-            self.mninfos = sorted(set(self.mninfos))
-
-        tmpSrcPath = self.srcfile + ".tmp"
-        tmpSrcFile = open(tmpSrcPath, 'w')
-        hdrrel = os.path.basename(self.hdrfile)
-
-        mnnames = []
-        mnmultinames = []
-        for item in self.mninfos:
-            mnfilename = os.path.basename(item)
-            mnfilename, ext = os.path.splitext(mnfilename)
-
-            if ext == ".info":
-                mnnames.append(mnfilename)
-            elif ext == ".mnmc":
-                mnmultinames.append(mnfilename)
-
-        tmpSrcFile.write(self.autogen)
-        tmpSrcFile.write("#include \"{0}\"\n\n".format(hdrrel))
-
-        if mnnames:
-            mndata = self.mnlistname + "_data"
-            tmpSrcFile.write("static const std::pair< TString, {0} > {1}[] = {{\n".format(self.mnlistelem, mndata))
-            for item in mnnames:
-                mnname = re.sub("[^-a-zA-Z0-9_]", "_", item)
-                tmpSrcFile.write("    std::make_pair(TString(\"{0}\"), &staticMn{1}{2}Ptr),\n".format(item, self.mnrankingSuffix, mnname))
-            tmpSrcFile.write("};\n")
-            tmpSrcFile.write("{0}({1},{1} + sizeof({1}) / sizeof({1}[0]));\n\n".format(self.mnlist, mndata))
-        else:
-            tmpSrcFile.write("{0};\n\n".format(self.mnlist))
-
-        if mnmultinames:
-            mnmultidata = self.mnmultilistname + "_data"
-            tmpSrcFile.write("static const std::pair< TString, {0} > {1}[] = {{\n".format(self.mnmultilistelem, mnmultidata))
-            for item in mnmultinames:
-                mnname = re.sub("[^-a-zA-Z0-9_]", "_", item)
-                tmpSrcFile.write("    std::make_pair(TString(\"{0}\"), &staticMnMulti{1}{2}Ptr),\n".format(item, self.mnrankingSuffix, mnname))
-            tmpSrcFile.write("};\n")
-            tmpSrcFile.write("{0}({1},{1} + sizeof({1}) / sizeof({1}[0]));\n".format(self.mnmultilist, mnmultidata))
-        else:
-            tmpSrcFile.write("{0};\n".format(self.mnmultilist))
-
-        tmpSrcFile.close()
-        shutil.move(tmpSrcPath, self.srcfile)
-
-
-def BuildMnsAllF(argv):
-    bldMns = BuildMns()
-    bldMns.InitForAll(argv)
-    bldMns.BuildMnsCpp()
-    bldMns.BuildMnsHeader()
-    bldMns.BuildMnFiles()
-
-
-def BuildMnsCppF(argv):
-    bldMns = BuildMns()
-    bldMns.InitForCpp(argv)
-    bldMns.BuildMnsCpp()
-
-
-def BuildMnsHeaderF(argv):
-    bldMns = BuildMns()
-    bldMns.InitForHeader(argv)
-    bldMns.BuildMnsHeader()
-
-
-def BuildMnsFilesF(argv):
-    bldMns = BuildMns()
-    bldMns.InitForFiles(argv)
-    bldMns.BuildMnFiles()
-
-
-def BuildMnF(argv):
-    bldMn = BuildMn()
-    bldMn.Run(argv)
-
-
-if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print >>sys.stderr, "Usage: build_mn.py <funcName> <args...>"
-        sys.exit(1)
-
-    if (sys.argv[2:]):
-        globals()[sys.argv[1]](sys.argv[2:])
-    else:
-        globals()[sys.argv[1]]()
diff --git a/build/scripts/build_pln_header.py b/build/scripts/build_pln_header.py
deleted file mode 100755
index c73693f444..0000000000
--- a/build/scripts/build_pln_header.py
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env python
-
-import sys
-import os
-
-
-def BuildPlnHeader():
-    if len(sys.argv) < 2:
-        print >>sys.stderr, "Usage: build_pln_header.py <absolute/path/to/OutFile>"
-        sys.exit(1)
-
-    print >>sys.stdout, "Build Pln Header..."
-    outPath = sys.argv[1]
-    tmpPath = outPath + '.tmp'
-    tmpFile = open(tmpPath, 'w')
-
-    tmpFile.write('#include <library/cpp/sse/sse.h>\n')
-    tmpFile.write('#include <kernel/relevfml/relev_fml.h>\n')
-    for path in sys.argv[2:]:
-        name = os.path.basename(path).split(".")[0]  # name without extensions
-        tmpFile.write('\nextern SRelevanceFormula fml{0};\n'.format(name))
-        tmpFile.write('float {0}(const float* f);\n'.format(name))
-        tmpFile.write('void {0}SSE(const float* const* factors, float* result);\n'.format(name))
-    tmpFile.close()
-    try:
-        os.remove(outPath)
-    except:
-        pass
-    try:
-        os.rename(tmpPath, outPath)
-    except:
-        print >>sys.stdout, 'Error: Failed to rename ' + tmpPath + ' to ' + outPath
-
-if __name__ == '__main__':
-    BuildPlnHeader()
diff --git a/build/scripts/gen_mx_table.py b/build/scripts/gen_mx_table.py
deleted file mode 100644
index 187c21c539..0000000000
--- a/build/scripts/gen_mx_table.py
+++ /dev/null
@@ -1,75 +0,0 @@
-import sys
-
-tmpl = """
-#include "yabs_mx_calc_table.h"
-
-#include <kernel/matrixnet/mn_sse.h>
-
-#include <library/cpp/archive/yarchive.h>
-
-#include <util/memory/blob.h>
-#include <util/generic/hash.h>
-#include <util/generic/ptr.h>
-#include <util/generic/singleton.h>
-
-using namespace NMatrixnet;
-
-extern "C" {
-    extern const unsigned char MxFormulas[];
-    extern const ui32 MxFormulasSize;
-}
-
-namespace {
-    struct TFml: public TBlob, public TMnSseInfo {
-        inline TFml(const TBlob& b)
-            : TBlob(b)
-            , TMnSseInfo(Data(), Size())
-        {
-        }
-    };
-
-    struct TFormulas: public THashMap<size_t, TAutoPtr<TFml>> {
-        inline TFormulas() {
-            TBlob b = TBlob::NoCopy(MxFormulas, MxFormulasSize);
-            TArchiveReader ar(b);
-            %s
-        }
-
-        inline const TMnSseInfo& at(size_t n) const noexcept {
-            return *find(n)->second;
-        }
-    };
-
-    %s
-
-    static func_descr_t yabs_funcs[] = {
-        %s
-    };
-}
-
-yabs_mx_calc_table_t yabs_mx_calc_table = {YABS_MX_CALC_VERSION, 10000, 0, yabs_funcs};
-"""
-
-if __name__ == '__main__':
-    init = []
-    body = []
-    defs = {}
-
-    for i in sys.argv[1:]:
-        name = i.replace('.', '_')
-        num = long(name.split('_')[1])
-
-        init.append('(*this)[%s] = new TFml(ar.ObjectBlobByKey("%s"));' % (num, '/' + i))
-
-        f1 = 'static void yabs_%s(size_t count, const float** args, double* res) {Singleton<TFormulas>()->at(%s).DoCalcRelevs(args, res, count);}' % (name, num)
-        f2 = 'static size_t yabs_%s_factor_count() {return Singleton<TFormulas>()->at(%s).MaxFactorIndex() + 1;}' % (name, num)
-
-        body.append(f1)
-        body.append(f2)
-
-        d1 = 'yabs_%s' % name
-        d2 = 'yabs_%s_factor_count' % name
-
-        defs[num] = '{%s, %s}' % (d1, d2)
-
-    print tmpl % ('\n'.join(init), '\n\n'.join(body), ',\n'.join((defs.get(i, '{nullptr, nullptr}') for i in range(0, 10000))))
diff --git a/build/scripts/perl_wrapper.py b/build/scripts/perl_wrapper.py
deleted file mode 100644
index cb4027f1d3..0000000000
--- a/build/scripts/perl_wrapper.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import os
-import sys
-import shutil
-
-if __name__ == '__main__':
-    path = sys.argv[1]
-    to = sys.argv[-1]
-    fr = sys.argv[-2]
-    to_dir = os.path.dirname(to)
-
-    os.chdir(to_dir)
-
-    f1 = os.path.basename(fr)
-    fr_ = os.path.dirname(fr)
-    f2 = os.path.basename(fr_)
-    fr_ = os.path.dirname(fr_)
-
-    os.makedirs(f2)
-    shutil.copyfile(fr, os.path.join(f2, f1))
-
-    if path[0] != '/':
-        path = os.path.join(os.path.dirname(__file__), path)
-
-    os.execv(path, [path] + sys.argv[2:])
diff --git a/build/scripts/ya.make b/build/scripts/ya.make
index 5448ec7db8..8cdf677024 100644
--- a/build/scripts/ya.make
+++ b/build/scripts/ya.make
@@ -8,8 +8,6 @@ IF (PY2)
         build_java_codenav_index.py
         build_java_with_error_prone.py
         build_java_with_error_prone2.py
-        build_mn.py
-        build_pln_header.py
         cat.py
         cgo1_wrapper.py
         check_config_h.py
@@ -44,7 +42,6 @@ IF (PY2)
         gen_aar_gradle_script.py
         gen_java_codenav_entry.py
         gen_java_codenav_protobuf.py
-        gen_mx_table.py
         gen_py3_reg.py
         gen_py_reg.py
         gen_test_apk_gradle_script.py
@@ -67,7 +64,6 @@ IF (PY2)
         mkver.py
         pack_ios.py
         pack_jcoverage_resources.py
-        perl_wrapper.py
         postprocess_go_fbs.py
         preprocess.py
         py_compile.py