Line data Source code
1 : // Copyright (c) 2012-2022 The Bitcoin Core developers
2 : // Distributed under the MIT software license, see the accompanying
3 : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 :
5 : #include <dbwrapper.h>
6 :
7 : #include <logging.h>
8 : #include <random.h>
9 : #include <serialize.h>
10 : #include <span.h>
11 : #include <streams.h>
12 : #include <util/fs.h>
13 : #include <util/fs_helpers.h>
14 : #include <util/strencodings.h>
15 :
16 : #include <algorithm>
17 : #include <cassert>
18 : #include <cstdarg>
19 : #include <cstdint>
20 : #include <cstdio>
21 : #include <leveldb/cache.h>
22 : #include <leveldb/db.h>
23 : #include <leveldb/env.h>
24 : #include <leveldb/filter_policy.h>
25 : #include <leveldb/helpers/memenv/memenv.h>
26 : #include <leveldb/iterator.h>
27 : #include <leveldb/options.h>
28 : #include <leveldb/slice.h>
29 : #include <leveldb/status.h>
30 : #include <leveldb/write_batch.h>
31 : #include <memory>
32 : #include <optional>
33 : #include <utility>
34 :
35 19272 : static auto CharCast(const std::byte* data) { return reinterpret_cast<const char*>(data); }
36 :
37 0 : bool DestroyDB(const std::string& path_str)
38 : {
39 0 : return leveldb::DestroyDB(path_str, {}).ok();
40 0 : }
41 :
42 : /** Handle database error by throwing dbwrapper_error exception.
43 : */
44 4 : static void HandleError(const leveldb::Status& status)
45 : {
46 4 : if (status.ok())
47 4 : return;
48 0 : const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
49 0 : LogPrintf("%s\n", errmsg);
50 0 : LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n");
51 0 : throw dbwrapper_error(errmsg);
52 0 : }
53 :
54 : class CBitcoinLevelDBLogger : public leveldb::Logger {
55 : public:
56 : // This code is adapted from posix_logger.h, which is why it is using vsprintf.
57 : // Please do not do this in normal code
58 3 : void Logv(const char * format, va_list ap) override {
59 3 : if (!LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug)) {
60 3 : return;
61 : }
62 : char buffer[500];
63 0 : for (int iter = 0; iter < 2; iter++) {
64 : char* base;
65 : int bufsize;
66 0 : if (iter == 0) {
67 0 : bufsize = sizeof(buffer);
68 0 : base = buffer;
69 0 : }
70 : else {
71 0 : bufsize = 30000;
72 0 : base = new char[bufsize];
73 : }
74 2 : char* p = base;
75 0 : char* limit = base + bufsize;
76 :
77 : // Print the message
78 0 : if (p < limit) {
79 : va_list backup_ap;
80 0 : va_copy(backup_ap, ap);
81 : // Do not use vsnprintf elsewhere in bitcoin source code, see above.
82 1 : p += vsnprintf(p, limit - p, format, backup_ap);
83 1 : va_end(backup_ap);
84 0 : }
85 1 :
86 : // Truncate to available space if necessary
87 0 : if (p >= limit) {
88 0 : if (iter == 0) {
89 0 : continue; // Try again with larger buffer
90 : }
91 : else {
92 0 : p = limit - 1;
93 : }
94 0 : }
95 :
96 : // Add newline if necessary
97 0 : if (p == base || p[-1] != '\n') {
98 0 : *p++ = '\n';
99 0 : }
100 :
101 0 : assert(p <= limit);
102 0 : base[std::min(bufsize - 1, (int)(p - base))] = '\0';
103 0 : LogPrintLevel(BCLog::LEVELDB, BCLog::Level::Debug, "%s", base); // NOLINT(bitcoin-unterminated-logprintf)
104 0 : if (base != buffer) {
105 0 : delete[] base;
106 0 : }
107 0 : break;
108 : }
109 3 : }
110 : };
111 :
112 3 : static void SetMaxOpenFiles(leveldb::Options *options) {
113 : // On most platforms the default setting of max_open_files (which is 1000)
114 : // is optimal. On Windows using a large file count is OK because the handles
115 : // do not interfere with select() loops. On 64-bit Unix hosts this value is
116 : // also OK, because up to that amount LevelDB will use an mmap
117 : // implementation that does not use extra file descriptors (the fds are
118 : // closed after being mmap'ed).
119 : //
120 : // Increasing the value beyond the default is dangerous because LevelDB will
121 : // fall back to a non-mmap implementation when the file count is too large.
122 : // On 32-bit Unix host we should decrease the value because the handles use
123 : // up real fds, and we want to avoid fd exhaustion issues.
124 : //
125 : // See PR #12495 for further discussion.
126 :
127 3 : int default_open_files = options->max_open_files;
128 : #ifndef WIN32
129 : if (sizeof(void*) < 8) {
130 : options->max_open_files = 64;
131 : }
132 : #endif
133 3 : LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
134 : options->max_open_files, default_open_files);
135 3 : }
136 :
137 3 : static leveldb::Options GetOptions(size_t nCacheSize)
138 : {
139 3 : leveldb::Options options;
140 3 : options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
141 3 : options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
142 3 : options.filter_policy = leveldb::NewBloomFilterPolicy(10);
143 3 : options.compression = leveldb::kNoCompression;
144 3 : options.info_log = new CBitcoinLevelDBLogger();
145 : if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
146 : // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
147 : // on corruption in later versions.
148 3 : options.paranoid_checks = true;
149 : }
150 3 : SetMaxOpenFiles(&options);
151 3 : return options;
152 : }
153 :
154 : struct CDBBatch::WriteBatchImpl {
155 : leveldb::WriteBatch batch;
156 : };
157 :
158 1 : CDBBatch::CDBBatch(const CDBWrapper& _parent)
159 1 : : parent{_parent},
160 2 : m_impl_batch{std::make_unique<CDBBatch::WriteBatchImpl>()} {};
161 :
162 1 : CDBBatch::~CDBBatch() = default;
163 :
164 0 : void CDBBatch::Clear()
165 : {
166 0 : m_impl_batch->batch.Clear();
167 0 : size_estimate = 0;
168 0 : }
169 :
170 1 : void CDBBatch::WriteImpl(Span<const std::byte> key, DataStream& ssValue)
171 : {
172 1 : leveldb::Slice slKey(CharCast(key.data()), key.size());
173 1 : ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent));
174 1 : leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size());
175 1 : m_impl_batch->batch.Put(slKey, slValue);
176 : // LevelDB serializes writes as:
177 : // - byte: header
178 : // - varint: key length (1 byte up to 127B, 2 bytes up to 16383B, ...)
179 : // - byte[]: key
180 : // - varint: value length
181 : // - byte[]: value
182 : // The formula below assumes the key and value are both less than 16k.
183 1 : size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size();
184 1 : }
185 :
186 0 : void CDBBatch::EraseImpl(Span<const std::byte> key)
187 : {
188 0 : leveldb::Slice slKey(CharCast(key.data()), key.size());
189 0 : m_impl_batch->batch.Delete(slKey);
190 : // LevelDB serializes erases as:
191 : // - byte: header
192 : // - varint: key length
193 : // - byte[]: key
194 : // The formula below assumes the key is less than 16kB.
195 0 : size_estimate += 2 + (slKey.size() > 127) + slKey.size();
196 0 : }
197 :
198 : struct LevelDBContext {
199 : //! custom environment this database is using (may be nullptr in case of default environment)
200 : leveldb::Env* penv;
201 :
202 : //! database options used
203 : leveldb::Options options;
204 :
205 : //! options used when reading from the database
206 : leveldb::ReadOptions readoptions;
207 :
208 : //! options used when iterating over values of the database
209 : leveldb::ReadOptions iteroptions;
210 :
211 : //! options used when writing to the database
212 : leveldb::WriteOptions writeoptions;
213 :
214 : //! options used when sync writing to the database
215 : leveldb::WriteOptions syncoptions;
216 :
217 : //! the database itself
218 : leveldb::DB* pdb;
219 : };
220 :
221 6 : CDBWrapper::CDBWrapper(const DBParams& params)
222 3 : : m_db_context{std::make_unique<LevelDBContext>()}, m_name{fs::PathToString(params.path.stem())}, m_path{params.path}, m_is_memory{params.memory_only}
223 : {
224 3 : DBContext().penv = nullptr;
225 3 : DBContext().readoptions.verify_checksums = true;
226 3 : DBContext().iteroptions.verify_checksums = true;
227 3 : DBContext().iteroptions.fill_cache = false;
228 3 : DBContext().syncoptions.sync = true;
229 3 : DBContext().options = GetOptions(params.cache_bytes);
230 3 : DBContext().options.create_if_missing = true;
231 3 : if (params.memory_only) {
232 3 : DBContext().penv = leveldb::NewMemEnv(leveldb::Env::Default());
233 3 : DBContext().options.env = DBContext().penv;
234 3 : } else {
235 0 : if (params.wipe_data) {
236 0 : LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(params.path));
237 0 : leveldb::Status result = leveldb::DestroyDB(fs::PathToString(params.path), DBContext().options);
238 0 : HandleError(result);
239 0 : }
240 0 : TryCreateDirectories(params.path);
241 0 : LogPrintf("Opening LevelDB in %s\n", fs::PathToString(params.path));
242 : }
243 : // PathToString() return value is safe to pass to leveldb open function,
244 : // because on POSIX leveldb passes the byte string directly to ::open(), and
245 : // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
246 : // (see env_posix.cc and env_windows.cc).
247 3 : leveldb::Status status = leveldb::DB::Open(DBContext().options, fs::PathToString(params.path), &DBContext().pdb);
248 3 : HandleError(status);
249 3 : LogPrintf("Opened LevelDB successfully\n");
250 :
251 3 : if (params.options.force_compact) {
252 0 : LogPrintf("Starting database compaction of %s\n", fs::PathToString(params.path));
253 0 : DBContext().pdb->CompactRange(nullptr, nullptr);
254 0 : LogPrintf("Finished database compaction of %s\n", fs::PathToString(params.path));
255 0 : }
256 :
257 : // The base-case obfuscation key, which is a noop.
258 3 : obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
259 :
260 3 : bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
261 :
262 3 : if (!key_exists && params.obfuscate && IsEmpty()) {
263 : // Initialize non-degenerate obfuscation if it won't upset
264 : // existing, non-obfuscated data.
265 1 : std::vector<unsigned char> new_key = CreateObfuscateKey();
266 :
267 : // Write `new_key` so we don't obfuscate the key with itself
268 1 : Write(OBFUSCATE_KEY_KEY, new_key);
269 1 : obfuscate_key = new_key;
270 :
271 1 : LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
272 1 : }
273 :
274 3 : LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(params.path), HexStr(obfuscate_key));
275 3 : }
276 :
277 3 : CDBWrapper::~CDBWrapper()
278 : {
279 3 : delete DBContext().pdb;
280 3 : DBContext().pdb = nullptr;
281 3 : delete DBContext().options.filter_policy;
282 3 : DBContext().options.filter_policy = nullptr;
283 3 : delete DBContext().options.info_log;
284 3 : DBContext().options.info_log = nullptr;
285 3 : delete DBContext().options.block_cache;
286 3 : DBContext().options.block_cache = nullptr;
287 3 : delete DBContext().penv;
288 3 : DBContext().options.env = nullptr;
289 3 : }
290 :
291 1 : bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
292 : {
293 1 : const bool log_memory = LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug);
294 1 : double mem_before = 0;
295 1 : if (log_memory) {
296 0 : mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
297 0 : }
298 1 : leveldb::Status status = DBContext().pdb->Write(fSync ? DBContext().syncoptions : DBContext().writeoptions, &batch.m_impl_batch->batch);
299 1 : HandleError(status);
300 1 : if (log_memory) {
301 0 : double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
302 0 : LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
303 : m_name, mem_before, mem_after);
304 0 : }
305 : return true;
306 1 : }
307 :
308 0 : size_t CDBWrapper::DynamicMemoryUsage() const
309 : {
310 0 : std::string memory;
311 0 : std::optional<size_t> parsed;
312 0 : if (!DBContext().pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) {
313 0 : LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
314 0 : return 0;
315 : }
316 0 : return parsed.value();
317 0 : }
318 :
319 : // Prefixed with null character to avoid collisions with other keys
320 : //
321 : // We must use a string constructor which specifies length so that we copy
322 : // past the null-terminator.
323 2 : const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
324 :
325 : const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
326 :
327 : /**
328 : * Returns a string (consisting of 8 random bytes) suitable for use as an
329 : * obfuscating XOR key.
330 : */
331 1 : std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
332 : {
333 1 : std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES);
334 1 : GetRandBytes(ret);
335 1 : return ret;
336 1 : }
337 :
338 19267 : std::optional<std::string> CDBWrapper::ReadImpl(Span<const std::byte> key) const
339 : {
340 19267 : leveldb::Slice slKey(CharCast(key.data()), key.size());
341 19267 : std::string strValue;
342 19267 : leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
343 19267 : if (!status.ok()) {
344 19267 : if (status.IsNotFound())
345 19267 : return std::nullopt;
346 0 : LogPrintf("LevelDB read failure: %s\n", status.ToString());
347 0 : HandleError(status);
348 0 : }
349 0 : return strValue;
350 19267 : }
351 :
352 1 : bool CDBWrapper::ExistsImpl(Span<const std::byte> key) const
353 : {
354 1 : leveldb::Slice slKey(CharCast(key.data()), key.size());
355 :
356 1 : std::string strValue;
357 1 : leveldb::Status status = DBContext().pdb->Get(DBContext().readoptions, slKey, &strValue);
358 1 : if (!status.ok()) {
359 1 : if (status.IsNotFound())
360 1 : return false;
361 0 : LogPrintf("LevelDB read failure: %s\n", status.ToString());
362 0 : HandleError(status);
363 0 : }
364 0 : return true;
365 1 : }
366 :
367 0 : size_t CDBWrapper::EstimateSizeImpl(Span<const std::byte> key1, Span<const std::byte> key2) const
368 : {
369 0 : leveldb::Slice slKey1(CharCast(key1.data()), key1.size());
370 0 : leveldb::Slice slKey2(CharCast(key2.data()), key2.size());
371 0 : uint64_t size = 0;
372 0 : leveldb::Range range(slKey1, slKey2);
373 0 : DBContext().pdb->GetApproximateSizes(&range, 1, &size);
374 0 : return size;
375 : }
376 :
377 1 : bool CDBWrapper::IsEmpty()
378 : {
379 1 : std::unique_ptr<CDBIterator> it(NewIterator());
380 1 : it->SeekToFirst();
381 1 : return !(it->Valid());
382 1 : }
383 :
384 : struct CDBIterator::IteratorImpl {
385 : const std::unique_ptr<leveldb::Iterator> iter;
386 :
387 3 : explicit IteratorImpl(leveldb::Iterator* _iter) : iter{_iter} {}
388 : };
389 :
390 3 : CDBIterator::CDBIterator(const CDBWrapper& _parent, std::unique_ptr<IteratorImpl> _piter) : parent(_parent),
391 3 : m_impl_iter(std::move(_piter)) {}
392 :
393 3 : CDBIterator* CDBWrapper::NewIterator()
394 : {
395 3 : return new CDBIterator{*this, std::make_unique<CDBIterator::IteratorImpl>(DBContext().pdb->NewIterator(DBContext().iteroptions))};
396 0 : }
397 :
398 2 : void CDBIterator::SeekImpl(Span<const std::byte> key)
399 : {
400 2 : leveldb::Slice slKey(CharCast(key.data()), key.size());
401 2 : m_impl_iter->iter->Seek(slKey);
402 2 : }
403 :
404 0 : Span<const std::byte> CDBIterator::GetKeyImpl() const
405 : {
406 0 : return MakeByteSpan(m_impl_iter->iter->key());
407 : }
408 :
409 0 : Span<const std::byte> CDBIterator::GetValueImpl() const
410 : {
411 0 : return MakeByteSpan(m_impl_iter->iter->value());
412 : }
413 :
414 3 : CDBIterator::~CDBIterator() = default;
415 3 : bool CDBIterator::Valid() const { return m_impl_iter->iter->Valid(); }
416 1 : void CDBIterator::SeekToFirst() { m_impl_iter->iter->SeekToFirst(); }
417 0 : void CDBIterator::Next() { m_impl_iter->iter->Next(); }
418 :
419 : namespace dbwrapper_private {
420 :
421 1 : const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
422 : {
423 1 : return w.obfuscate_key;
424 : }
425 :
426 : } // namespace dbwrapper_private
|
