1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
|
#pragma clang system_header
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// This module defines an abstract interface for iterating through pages in a
// Parquet column chunk within a row group. It could be extended in the future
// to iterate through all data pages in all chunks in a file.
#pragma once
#include <memory>
#include <string>
#include <unordered_map>
#include <gtest/gtest.h>
#include "contrib/libs/apache/arrow_next/cpp/src/arrow/filesystem/filesystem.h"
#include "contrib/libs/apache/arrow_next/cpp/src/arrow/filesystem/localfs.h"
#include "contrib/libs/apache/arrow_next/cpp/src/arrow/status.h"
#include "contrib/libs/apache/arrow_next/cpp/src/arrow/util/io_util.h"
#include "contrib/libs/apache/arrow_next/cpp/src/parquet/encryption/encryption.h"
#include "contrib/libs/apache/arrow_next/cpp/src/parquet/test_util.h"
namespace parquet20 {
class ParquetFileReader;
namespace encryption::test {
using ::arrow20::internal::TemporaryDir;
constexpr int kFixedLength = 10;
const char kFooterEncryptionKey[] = "0123456789012345"; // 128bit/16
const char kColumnEncryptionKey1[] = "1234567890123450";
const char kColumnEncryptionKey2[] = "1234567890123451";
const char kFileName[] = "tester";
// Get the path of file inside parquet test data directory
std::string data_file(const char* file);
// A temporary directory that contains the encrypted files generated in the tests.
extern std::unique_ptr<TemporaryDir> temp_dir;
inline ::arrow20::Result<std::unique_ptr<TemporaryDir>> temp_data_dir() {
return TemporaryDir::Make("parquet-encryption-test-");
}
const char kDoubleFieldName[] = "double_field";
const char kFloatFieldName[] = "float_field";
const char kBooleanFieldName[] = "boolean_field";
const char kInt32FieldName[] = "int32_field";
const char kInt64FieldName[] = "int64_field";
const char kInt96FieldName[] = "int96_field";
const char kByteArrayFieldName[] = "ba_field";
const char kFixedLenByteArrayFieldName[] = "flba_field";
const char kFooterMasterKey[] = "0123456789012345";
const char kFooterMasterKeyId[] = "kf";
const char* const kColumnMasterKeys[] = {"1234567890123450", "1234567890123451",
"1234567890123452", "1234567890123453",
"1234567890123454", "1234567890123455"};
const char* const kColumnMasterKeyIds[] = {"kc1", "kc2", "kc3", "kc4", "kc5", "kc6"};
// New master key values used to simulate key rotation
const char kNewFooterMasterKey[] = "9123456789012345";
const char* const kNewColumnMasterKeys[] = {"9234567890123450", "9234567890123451",
"9234567890123452", "9234567890123453",
"9234567890123454", "9234567890123455"};
// The result of this function will be used to set into TestOnlyInMemoryKmsClientFactory
// as the key mapping to look at.
std::unordered_map<std::string, std::string> BuildKeyMap(const char* const* column_ids,
const char* const* column_keys,
const char* footer_id,
const char* footer_key);
// The result of this function will be used to set into EncryptionConfiguration
// as column keys.
std::string BuildColumnKeyMapping();
// FileEncryptor and FileDecryptor are helper classes to write/read an encrypted parquet
// file corresponding to each pair of FileEncryptionProperties/FileDecryptionProperties.
// FileEncryptor writes the file with fixed data values and FileDecryptor reads the file
// and verify the correctness of data values.
class FileEncryptor {
public:
FileEncryptor();
void EncryptFile(
std::string file,
std::shared_ptr<parquet20::FileEncryptionProperties> encryption_configurations);
private:
std::shared_ptr<schema::GroupNode> SetupEncryptionSchema();
int num_rowgroups_ = 5;
int rows_per_rowgroup_ = 50;
std::shared_ptr<schema::GroupNode> schema_;
};
class FileDecryptor {
public:
void DecryptFile(
const std::string& file_name,
const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
void DecryptPageIndex(
const std::string& file_name,
const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
private:
void CheckFile(
parquet20::ParquetFileReader* file_reader,
const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
void CheckPageIndex(
parquet20::ParquetFileReader* file_reader,
const std::shared_ptr<FileDecryptionProperties>& file_decryption_properties);
};
} // namespace encryption::test
} // namespace parquet20
|
