Line data Source code
1 : // Copyright (c) 2012-2021 The Bitcoin Core developers
2 : // Distributed under the MIT software license, see the accompanying
3 : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 :
5 : #include <dbwrapper.h>
6 :
7 : #include <fs.h>
8 : #include <logging.h>
9 : #include <random.h>
10 : #include <util/strencodings.h>
11 : #include <util/system.h>
12 :
13 : #include <algorithm>
14 : #include <cassert>
15 : #include <cstdarg>
16 : #include <cstdint>
17 : #include <cstdio>
18 : #include <leveldb/cache.h>
19 : #include <leveldb/db.h>
20 : #include <leveldb/env.h>
21 : #include <leveldb/filter_policy.h>
22 : #include <leveldb/helpers/memenv/memenv.h>
23 : #include <leveldb/iterator.h>
24 : #include <leveldb/options.h>
25 : #include <leveldb/status.h>
26 : #include <memory>
27 : #include <optional>
28 :
29 : class CBitcoinLevelDBLogger : public leveldb::Logger {
30 : public:
31 : // This code is adapted from posix_logger.h, which is why it is using vsprintf.
32 : // Please do not do this in normal code
33 91524 : void Logv(const char * format, va_list ap) override {
34 91524 : if (!LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug)) {
35 91524 : return;
36 : }
37 : char buffer[500];
38 0 : for (int iter = 0; iter < 2; iter++) {
39 : char* base;
40 : int bufsize;
41 0 : if (iter == 0) {
42 0 : bufsize = sizeof(buffer);
43 0 : base = buffer;
44 0 : }
45 : else {
46 0 : bufsize = 30000;
47 0 : base = new char[bufsize];
48 : }
49 0 : char* p = base;
50 0 : char* limit = base + bufsize;
51 :
52 : // Print the message
53 0 : if (p < limit) {
54 : va_list backup_ap;
55 0 : va_copy(backup_ap, ap);
56 : // Do not use vsnprintf elsewhere in bitcoin source code, see above.
57 0 : p += vsnprintf(p, limit - p, format, backup_ap);
58 0 : va_end(backup_ap);
59 0 : }
60 :
61 : // Truncate to available space if necessary
62 0 : if (p >= limit) {
63 0 : if (iter == 0) {
64 0 : continue; // Try again with larger buffer
65 : }
66 : else {
67 0 : p = limit - 1;
68 : }
69 0 : }
70 :
71 : // Add newline if necessary
72 0 : if (p == base || p[-1] != '\n') {
73 0 : *p++ = '\n';
74 0 : }
75 :
76 0 : assert(p <= limit);
77 0 : base[std::min(bufsize - 1, (int)(p - base))] = '\0';
78 0 : LogPrintLevel(BCLog::LEVELDB, BCLog::Level::Debug, "%s", base); /* Continued */
79 0 : if (base != buffer) {
80 0 : delete[] base;
81 0 : }
82 0 : break;
83 : }
84 91524 : }
85 : };
86 :
87 25143 : static void SetMaxOpenFiles(leveldb::Options *options) {
88 : // On most platforms the default setting of max_open_files (which is 1000)
89 : // is optimal. On Windows using a large file count is OK because the handles
90 : // do not interfere with select() loops. On 64-bit Unix hosts this value is
91 : // also OK, because up to that amount LevelDB will use an mmap
92 : // implementation that does not use extra file descriptors (the fds are
93 : // closed after being mmap'ed).
94 : //
95 : // Increasing the value beyond the default is dangerous because LevelDB will
96 : // fall back to a non-mmap implementation when the file count is too large.
97 : // On 32-bit Unix host we should decrease the value because the handles use
98 : // up real fds, and we want to avoid fd exhaustion issues.
99 : //
100 : // See PR #12495 for further discussion.
101 :
102 25143 : int default_open_files = options->max_open_files;
103 : #ifndef WIN32
104 : if (sizeof(void*) < 8) {
105 : options->max_open_files = 64;
106 : }
107 : #endif
108 25143 : LogPrint(BCLog::LEVELDB, "LevelDB using max_open_files=%d (default=%d)\n",
109 : options->max_open_files, default_open_files);
110 25143 : }
111 :
112 25143 : static leveldb::Options GetOptions(size_t nCacheSize)
113 : {
114 25143 : leveldb::Options options;
115 25143 : options.block_cache = leveldb::NewLRUCache(nCacheSize / 2);
116 25143 : options.write_buffer_size = nCacheSize / 4; // up to two write buffers may be held in memory simultaneously
117 25143 : options.filter_policy = leveldb::NewBloomFilterPolicy(10);
118 25143 : options.compression = leveldb::kNoCompression;
119 25143 : options.info_log = new CBitcoinLevelDBLogger();
120 : if (leveldb::kMajorVersion > 1 || (leveldb::kMajorVersion == 1 && leveldb::kMinorVersion >= 16)) {
121 : // LevelDB versions before 1.16 consider short writes to be corruption. Only trigger error
122 : // on corruption in later versions.
123 25143 : options.paranoid_checks = true;
124 : }
125 25143 : SetMaxOpenFiles(&options);
126 25143 : return options;
127 : }
128 :
129 68095 : CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
130 25147 : : m_name{fs::PathToString(path.stem())}
131 17809 : {
132 25143 : penv = nullptr;
133 25143 : readoptions.verify_checksums = true;
134 25143 : iteroptions.verify_checksums = true;
135 25143 : iteroptions.fill_cache = false;
136 25143 : syncoptions.sync = true;
137 25143 : options = GetOptions(nCacheSize);
138 25143 : options.create_if_missing = true;
139 25143 : if (fMemory) {
140 2166 : penv = leveldb::NewMemEnv(leveldb::Env::Default());
141 2166 : options.env = penv;
142 2166 : } else {
143 22977 : if (fWipe) {
144 1962 : LogPrintf("Wiping LevelDB in %s\n", fs::PathToString(path));
145 1962 : leveldb::Status result = leveldb::DestroyDB(fs::PathToString(path), options);
146 1962 : dbwrapper_private::HandleError(result);
147 1962 : }
148 22977 : TryCreateDirectories(path);
149 22977 : LogPrintf("Opening LevelDB in %s\n", fs::PathToString(path));
150 : }
151 : // PathToString() return value is safe to pass to leveldb open function,
152 : // because on POSIX leveldb passes the byte string directly to ::open(), and
153 : // on Windows it converts from UTF-8 to UTF-16 before calling ::CreateFileW
154 : // (see env_posix.cc and env_windows.cc).
155 25143 : leveldb::Status status = leveldb::DB::Open(options, fs::PathToString(path), &pdb);
156 25143 : dbwrapper_private::HandleError(status);
157 25139 : LogPrintf("Opened LevelDB successfully\n");
158 :
159 25139 : if (gArgs.GetBoolArg("-forcecompactdb", false)) {
160 0 : LogPrintf("Starting database compaction of %s\n", fs::PathToString(path));
161 0 : pdb->CompactRange(nullptr, nullptr);
162 0 : LogPrintf("Finished database compaction of %s\n", fs::PathToString(path));
163 0 : }
164 :
165 : // The base-case obfuscation key, which is a noop.
166 25139 : obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
167 :
168 25139 : bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
169 :
170 25139 : if (!key_exists && obfuscate && IsEmpty()) {
171 : // Initialize non-degenerate obfuscation if it won't upset
172 : // existing, non-obfuscated data.
173 1090 : std::vector<unsigned char> new_key = CreateObfuscateKey();
174 :
175 : // Write `new_key` so we don't obfuscate the key with itself
176 1090 : Write(OBFUSCATE_KEY_KEY, new_key);
177 1090 : obfuscate_key = new_key;
178 :
179 1090 : LogPrintf("Wrote new obfuscate key for %s: %s\n", fs::PathToString(path), HexStr(obfuscate_key));
180 1090 : }
181 :
182 25139 : LogPrintf("Using obfuscation key for %s: %s\n", fs::PathToString(path), HexStr(obfuscate_key));
183 42952 : }
184 :
185 42944 : CDBWrapper::~CDBWrapper()
186 17806 : {
187 25138 : delete pdb;
188 25138 : pdb = nullptr;
189 25138 : delete options.filter_policy;
190 25138 : options.filter_policy = nullptr;
191 25138 : delete options.info_log;
192 25138 : options.info_log = nullptr;
193 25138 : delete options.block_cache;
194 25138 : options.block_cache = nullptr;
195 25138 : delete penv;
196 25138 : options.env = nullptr;
197 42944 : }
198 :
199 709934 : bool CDBWrapper::WriteBatch(CDBBatch& batch, bool fSync)
200 : {
201 709934 : const bool log_memory = LogAcceptCategory(BCLog::LEVELDB, BCLog::Level::Debug);
202 709934 : double mem_before = 0;
203 709934 : if (log_memory) {
204 0 : mem_before = DynamicMemoryUsage() / 1024.0 / 1024;
205 0 : }
206 709934 : leveldb::Status status = pdb->Write(fSync ? syncoptions : writeoptions, &batch.batch);
207 709934 : dbwrapper_private::HandleError(status);
208 709934 : if (log_memory) {
209 0 : double mem_after = DynamicMemoryUsage() / 1024.0 / 1024;
210 0 : LogPrint(BCLog::LEVELDB, "WriteBatch memory usage: db=%s, before=%.1fMiB, after=%.1fMiB\n",
211 : m_name, mem_before, mem_after);
212 0 : }
213 : return true;
214 709934 : }
215 :
216 0 : size_t CDBWrapper::DynamicMemoryUsage() const
217 : {
218 0 : std::string memory;
219 0 : std::optional<size_t> parsed;
220 0 : if (!pdb->GetProperty("leveldb.approximate-memory-usage", &memory) || !(parsed = ToIntegral<size_t>(memory))) {
221 0 : LogPrint(BCLog::LEVELDB, "Failed to get approximate-memory-usage property\n");
222 0 : return 0;
223 : }
224 0 : return parsed.value();
225 0 : }
226 :
227 : // Prefixed with null character to avoid collisions with other keys
228 : //
229 : // We must use a string constructor which specifies length so that we copy
230 : // past the null-terminator.
231 : const std::string CDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
232 :
233 : const unsigned int CDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
234 :
235 : /**
236 : * Returns a string (consisting of 8 random bytes) suitable for use as an
237 : * obfuscating XOR key.
238 : */
239 1090 : std::vector<unsigned char> CDBWrapper::CreateObfuscateKey() const
240 : {
241 1090 : std::vector<uint8_t> ret(OBFUSCATE_KEY_NUM_BYTES);
242 1090 : GetRandBytes(ret);
243 1090 : return ret;
244 1090 : }
245 :
246 5455 : bool CDBWrapper::IsEmpty()
247 : {
248 5455 : std::unique_ptr<CDBIterator> it(NewIterator());
249 5455 : it->SeekToFirst();
250 5455 : return !(it->Valid());
251 5455 : }
252 :
253 4017732 : CDBIterator::~CDBIterator() { delete piter; }
254 8575588 : bool CDBIterator::Valid() const { return piter->Valid(); }
255 5455 : void CDBIterator::SeekToFirst() { piter->SeekToFirst(); }
256 660060 : void CDBIterator::Next() { piter->Next(); }
257 :
258 : namespace dbwrapper_private {
259 :
260 737040 : void HandleError(const leveldb::Status& status)
261 : {
262 737040 : if (status.ok())
263 737034 : return;
264 12 : const std::string errmsg = "Fatal LevelDB error: " + status.ToString();
265 6 : LogPrintf("%s\n", errmsg);
266 6 : LogPrintf("You can use -debug=leveldb to get more complete diagnostic messages\n");
267 6 : throw dbwrapper_error(errmsg);
268 6 : }
269 :
270 2793530 : const std::vector<unsigned char>& GetObfuscateKey(const CDBWrapper &w)
271 : {
272 2793530 : return w.obfuscate_key;
273 : }
274 :
275 : } // namespace dbwrapper_private
|
