aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorudovichenko-r <rvu@ydb.tech>2022-12-06 14:27:43 +0300
committerudovichenko-r <rvu@ydb.tech>2022-12-06 14:27:43 +0300
commitc98004737b7c07ca8fc7d840690738cd1d405ed2 (patch)
tree6e52387d42360d6eb35f77763a42399f9bd02777
parenta13a044a719875aa309bee399534784dfe876d8c (diff)
downloadydb-c98004737b7c07ca8fc7d840690738cd1d405ed2.tar.gz
[dq] Remove wrong file dedup
-rw-r--r--ydb/library/yql/providers/dq/actors/executer_actor.cpp2
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_gateway.h4
-rw-r--r--ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp7
3 files changed, 3 insertions, 10 deletions
diff --git a/ydb/library/yql/providers/dq/actors/executer_actor.cpp b/ydb/library/yql/providers/dq/actors/executer_actor.cpp
index 077e4248c7e..a0f7855f1eb 100644
--- a/ydb/library/yql/providers/dq/actors/executer_actor.cpp
+++ b/ydb/library/yql/providers/dq/actors/executer_actor.cpp
@@ -361,7 +361,7 @@ private:
YQL_CLOG(DEBUG, ProviderDq) << "WorkerInfo: " << NDqs::NExecutionHelpers::PrettyPrintWorkerInfo(workerInfo, taskMeta.GetStageId());
YQL_CLOG(DEBUG, ProviderDq) << "TaskInfo: " << i << "/" << tasks[i].GetId();
for (const auto& file: taskMeta.GetFiles()) {
- YQL_CLOG(DEBUG, ProviderDq) << " ObjectId: " << file.GetObjectId();
+ YQL_CLOG(DEBUG, ProviderDq) << " ObjectId: " << file.GetObjectId() << ", " << file.GetName() << ", " << file.GetLocalPath();
}
i++;
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h
index 2a5904ec6e8..efa1eb17060 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h
@@ -30,13 +30,13 @@ public:
struct TFileResourceHash {
std::size_t operator()(const TFileResource& f) const {
- return std::hash<TString>()(f.GetObjectId());
+ return std::hash<TString>()(f.GetName());
}
};
struct TFileResourceEqual {
bool operator()(const TFileResource& a, const TFileResource& b) const {
- return a.GetObjectId() == b.GetObjectId();
+ return a.GetName() == b.GetName();
}
};
diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp
index 18d38fae28b..fda87142148 100644
--- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp
+++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp
@@ -1751,16 +1751,9 @@ private:
task.GetMeta().UnpackTo(&taskMeta);
auto* files = taskMeta.MutableFiles();
- THashSet<TString> initialized;
for (auto& file : *files) {
if (file.GetObjectType() != Yql::DqsProto::TFile::EEXE_FILE) {
- if (initialized.contains(file.GetObjectId())) {
- file.SetObjectType(Yql::DqsProto::TFile::EEXE_FILE); // skip this in runner
- continue;
- }
- initialized.insert(file.GetObjectId());
-
auto maybeFile = FileCache->FindFile(file.GetObjectId());
if (!maybeFile) {
throw std::runtime_error("Cannot find object `" + file.GetObjectId() + "' in cache");