# HG changeset patch # Parent 5de7eafc3ceca2196d84d5b6106e01046efda034 diff --git a/security/manager/ssl/nsNSSComponent.cpp b/security/manager/ssl/nsNSSComponent.cpp --- a/security/manager/ssl/nsNSSComponent.cpp +++ b/security/manager/ssl/nsNSSComponent.cpp @@ -13,7 +13,6 @@ #include "SharedSSLState.h" #include "cert.h" #include "certdb.h" -#include "mozStorageCID.h" #include "mozilla/ArrayUtils.h" #include "mozilla/Assertions.h" #include "mozilla/Casting.h" @@ -2038,14 +2037,6 @@ nsNSSComponent::Init() return NS_ERROR_NOT_AVAILABLE; } - // To avoid a sqlite3_config race in NSS init, as a workaround for - // bug 730495, we require the storage service to get initialized first. - nsCOMPtr storageService = - do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID); - if (!storageService) { - return NS_ERROR_NOT_AVAILABLE; - } - MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("Beginning NSS initialization\n")); nsresult rv = InitializePIPNSSBundle(); diff --git a/storage/TelemetryVFS.cpp b/storage/TelemetryVFS.cpp --- a/storage/TelemetryVFS.cpp +++ b/storage/TelemetryVFS.cpp @@ -834,6 +834,11 @@ xNextSystemCall(sqlite3_vfs *vfs, const namespace mozilla { namespace storage { +const char *GetVFSName() +{ + return "telemetry-vfs"; +} + sqlite3_vfs* ConstructTelemetryVFS() { #if defined(XP_WIN) @@ -867,7 +872,7 @@ sqlite3_vfs* ConstructTelemetryVFS() MOZ_ASSERT(vfs->iVersion <= LAST_KNOWN_VFS_VERSION); tvfs->szOsFile = sizeof(telemetry_file) - sizeof(sqlite3_file) + vfs->szOsFile; tvfs->mxPathname = vfs->mxPathname; - tvfs->zName = "telemetry-vfs"; + tvfs->zName = GetVFSName(); tvfs->pAppData = vfs; tvfs->xOpen = xOpen; tvfs->xDelete = xDelete; diff --git a/storage/mozStorageConnection.cpp b/storage/mozStorageConnection.cpp --- a/storage/mozStorageConnection.cpp +++ b/storage/mozStorageConnection.cpp @@ -73,6 +73,8 @@ namespace storage { using mozilla::dom::quota::QuotaObject; +const char *GetVFSName(); + namespace { int @@ -627,7 +629,7 @@ Connection::initialize() AUTO_PROFILER_LABEL("Connection::initialize", STORAGE); // in memory database requested, sqlite uses a magic file name - int srv = ::sqlite3_open_v2(":memory:", &mDBConn, mFlags, nullptr); + int srv = ::sqlite3_open_v2(":memory:", &mDBConn, mFlags, GetVFSName()); if (srv != SQLITE_OK) { mDBConn = nullptr; return convertResultCode(srv); @@ -660,7 +662,7 @@ Connection::initialize(nsIFile *aDatabas #else static const char* sIgnoreLockingVFS = "unix-none"; #endif - const char* vfs = mIgnoreLockingMode ? sIgnoreLockingVFS : nullptr; + const char* vfs = mIgnoreLockingMode ? sIgnoreLockingVFS : GetVFSName(); int srv = ::sqlite3_open_v2(NS_ConvertUTF16toUTF8(path).get(), &mDBConn, mFlags, vfs); @@ -694,7 +696,7 @@ Connection::initialize(nsIFileURL *aFile rv = aFileURL->GetSpec(spec); NS_ENSURE_SUCCESS(rv, rv); - int srv = ::sqlite3_open_v2(spec.get(), &mDBConn, mFlags, nullptr); + int srv = ::sqlite3_open_v2(spec.get(), &mDBConn, mFlags, GetVFSName()); if (srv != SQLITE_OK) { mDBConn = nullptr; return convertResultCode(srv); diff --git a/storage/mozStorageService.cpp b/storage/mozStorageService.cpp --- a/storage/mozStorageService.cpp +++ b/storage/mozStorageService.cpp @@ -24,6 +24,7 @@ #include "mozIStoragePendingStatement.h" #include "sqlite3.h" +#include "mozilla/AutoSQLiteLifetime.h" #ifdef SQLITE_OS_WIN // "windows.h" was included and it can #define lots of things we care about... @@ -32,13 +33,6 @@ #include "nsIPromptService.h" -#ifdef MOZ_STORAGE_MEMORY -# include "mozmemory.h" -# ifdef MOZ_DMD -# include "DMD.h" -# endif -#endif - //////////////////////////////////////////////////////////////////////////////// //// Defines @@ -282,12 +276,6 @@ Service::~Service() if (rc != SQLITE_OK) NS_WARNING("Failed to unregister sqlite vfs wrapper."); - // Shutdown the sqlite3 API. Warn if shutdown did not turn out okay, but - // there is nothing actionable we can do in that case. - rc = ::sqlite3_shutdown(); - if (rc != SQLITE_OK) - NS_WARNING("sqlite3 did not shutdown cleanly."); - shutdown(); // To release sXPConnect. gService = nullptr; @@ -400,121 +388,7 @@ Service::shutdown() } sqlite3_vfs *ConstructTelemetryVFS(); - -#ifdef MOZ_STORAGE_MEMORY - -namespace { - -// By default, SQLite tracks the size of all its heap blocks by adding an extra -// 8 bytes at the start of the block to hold the size. Unfortunately, this -// causes a lot of 2^N-sized allocations to be rounded up by jemalloc -// allocator, wasting memory. For example, a request for 1024 bytes has 8 -// bytes added, becoming a request for 1032 bytes, and jemalloc rounds this up -// to 2048 bytes, wasting 1012 bytes. (See bug 676189 for more details.) -// -// So we register jemalloc as the malloc implementation, which avoids this -// 8-byte overhead, and thus a lot of waste. This requires us to provide a -// function, sqliteMemRoundup(), which computes the actual size that will be -// allocated for a given request. SQLite uses this function before all -// allocations, and may be able to use any excess bytes caused by the rounding. -// -// Note: the wrappers for malloc, realloc and moz_malloc_usable_size are -// necessary because the sqlite_mem_methods type signatures differ slightly -// from the standard ones -- they use int instead of size_t. But we don't need -// a wrapper for free. - -#ifdef MOZ_DMD - -// sqlite does its own memory accounting, and we use its numbers in our memory -// reporters. But we don't want sqlite's heap blocks to show up in DMD's -// output as unreported, so we mark them as reported when they're allocated and -// mark them as unreported when they are freed. -// -// In other words, we are marking all sqlite heap blocks as reported even -// though we're not reporting them ourselves. Instead we're trusting that -// sqlite is fully and correctly accounting for all of its heap blocks via its -// own memory accounting. Well, we don't have to trust it entirely, because -// it's easy to keep track (while doing this DMD-specific marking) of exactly -// how much memory SQLite is using. And we can compare that against what -// SQLite reports it is using. - -MOZ_DEFINE_MALLOC_SIZE_OF_ON_ALLOC(SqliteMallocSizeOfOnAlloc) -MOZ_DEFINE_MALLOC_SIZE_OF_ON_FREE(SqliteMallocSizeOfOnFree) - -#endif - -static void *sqliteMemMalloc(int n) -{ - void* p = ::malloc(n); -#ifdef MOZ_DMD - gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(p); -#endif - return p; -} - -static void sqliteMemFree(void *p) -{ -#ifdef MOZ_DMD - gSqliteMemoryUsed -= SqliteMallocSizeOfOnFree(p); -#endif - ::free(p); -} - -static void *sqliteMemRealloc(void *p, int n) -{ -#ifdef MOZ_DMD - gSqliteMemoryUsed -= SqliteMallocSizeOfOnFree(p); - void *pnew = ::realloc(p, n); - if (pnew) { - gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(pnew); - } else { - // realloc failed; undo the SqliteMallocSizeOfOnFree from above - gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(p); - } - return pnew; -#else - return ::realloc(p, n); -#endif -} - -static int sqliteMemSize(void *p) -{ - return ::moz_malloc_usable_size(p); -} - -static int sqliteMemRoundup(int n) -{ - n = malloc_good_size(n); - - // jemalloc can return blocks of size 2 and 4, but SQLite requires that all - // allocations be 8-aligned. So we round up sub-8 requests to 8. This - // wastes a small amount of memory but is obviously safe. - return n <= 8 ? 8 : n; -} - -static int sqliteMemInit(void *p) -{ - return 0; -} - -static void sqliteMemShutdown(void *p) -{ -} - -const sqlite3_mem_methods memMethods = { - &sqliteMemMalloc, - &sqliteMemFree, - &sqliteMemRealloc, - &sqliteMemSize, - &sqliteMemRoundup, - &sqliteMemInit, - &sqliteMemShutdown, - nullptr -}; - -} // namespace - -#endif // MOZ_STORAGE_MEMORY +const char *GetVFSName(); static const char* sObserverTopics[] = { "memory-pressure", @@ -527,28 +401,13 @@ Service::initialize() { MOZ_ASSERT(NS_IsMainThread(), "Must be initialized on the main thread"); - int rc; - -#ifdef MOZ_STORAGE_MEMORY - rc = ::sqlite3_config(SQLITE_CONFIG_MALLOC, &memMethods); - if (rc != SQLITE_OK) - return convertResultCode(rc); -#endif - - // TODO (bug 1191405): do not preallocate the connections caches until we - // have figured the impact on our consumers and memory. - sqlite3_config(SQLITE_CONFIG_PAGECACHE, NULL, 0, 0); - - // Explicitly initialize sqlite3. Although this is implicitly called by - // various sqlite3 functions (and the sqlite3_open calls in our case), - // the documentation suggests calling this directly. So we do. - rc = ::sqlite3_initialize(); + int rc = AutoSQLiteLifetime::getInitResult(); if (rc != SQLITE_OK) return convertResultCode(rc); mSqliteVFS = ConstructTelemetryVFS(); if (mSqliteVFS) { - rc = sqlite3_vfs_register(mSqliteVFS, 1); + rc = sqlite3_vfs_register(mSqliteVFS, 0); if (rc != SQLITE_OK) return convertResultCode(rc); } else { diff --git a/toolkit/xre/Bootstrap.cpp b/toolkit/xre/Bootstrap.cpp --- a/toolkit/xre/Bootstrap.cpp +++ b/toolkit/xre/Bootstrap.cpp @@ -6,11 +6,15 @@ #include "mozilla/Bootstrap.h" #include "nsXPCOM.h" +#include "AutoSQLiteLifetime.h" + namespace mozilla { class BootstrapImpl final : public Bootstrap { protected: + AutoSQLiteLifetime mSQLLT; + virtual void Dispose() override { delete this; diff --git a/toolkit/xre/moz.build b/toolkit/xre/moz.build --- a/toolkit/xre/moz.build +++ b/toolkit/xre/moz.build @@ -30,7 +30,7 @@ EXPORTS += [ 'nsIAppStartupNotifier.h', ] -EXPORTS.mozilla += ['Bootstrap.h'] +EXPORTS.mozilla += ['AutoSQLiteLifetime.h', 'Bootstrap.h'] if CONFIG['MOZ_INSTRUMENT_EVENT_LOOP']: EXPORTS += ['EventTracer.h'] @@ -91,6 +91,7 @@ if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'andr ] UNIFIED_SOURCES += [ + 'AutoSQLiteLifetime.cpp', 'Bootstrap.cpp', 'CreateAppData.cpp', 'nsAppStartupNotifier.cpp', diff --git a/toolkit/xre/AutoSQLiteLifetime.cpp b/toolkit/xre/AutoSQLiteLifetime.cpp new file mode 100644 --- /dev/null +++ b/toolkit/xre/AutoSQLiteLifetime.cpp @@ -0,0 +1,167 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsDebug.h" +#include "AutoSQLiteLifetime.h" +#include "sqlite3.h" + +#ifdef MOZ_STORAGE_MEMORY +# include "mozmemory.h" +# ifdef MOZ_DMD +# include "DMD.h" +# endif + +namespace { + +// By default, SQLite tracks the size of all its heap blocks by adding an extra +// 8 bytes at the start of the block to hold the size. Unfortunately, this +// causes a lot of 2^N-sized allocations to be rounded up by jemalloc +// allocator, wasting memory. For example, a request for 1024 bytes has 8 +// bytes added, becoming a request for 1032 bytes, and jemalloc rounds this up +// to 2048 bytes, wasting 1012 bytes. (See bug 676189 for more details.) +// +// So we register jemalloc as the malloc implementation, which avoids this +// 8-byte overhead, and thus a lot of waste. This requires us to provide a +// function, sqliteMemRoundup(), which computes the actual size that will be +// allocated for a given request. SQLite uses this function before all +// allocations, and may be able to use any excess bytes caused by the rounding. +// +// Note: the wrappers for malloc, realloc and moz_malloc_usable_size are +// necessary because the sqlite_mem_methods type signatures differ slightly +// from the standard ones -- they use int instead of size_t. But we don't need +// a wrapper for free. + +#ifdef MOZ_DMD + +// sqlite does its own memory accounting, and we use its numbers in our memory +// reporters. But we don't want sqlite's heap blocks to show up in DMD's +// output as unreported, so we mark them as reported when they're allocated and +// mark them as unreported when they are freed. +// +// In other words, we are marking all sqlite heap blocks as reported even +// though we're not reporting them ourselves. Instead we're trusting that +// sqlite is fully and correctly accounting for all of its heap blocks via its +// own memory accounting. Well, we don't have to trust it entirely, because +// it's easy to keep track (while doing this DMD-specific marking) of exactly +// how much memory SQLite is using. And we can compare that against what +// SQLite reports it is using. + +MOZ_DEFINE_MALLOC_SIZE_OF_ON_ALLOC(SqliteMallocSizeOfOnAlloc) +MOZ_DEFINE_MALLOC_SIZE_OF_ON_FREE(SqliteMallocSizeOfOnFree) + +#endif + +static void *sqliteMemMalloc(int n) +{ + void* p = ::malloc(n); +#ifdef MOZ_DMD + gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(p); +#endif + return p; +} + +static void sqliteMemFree(void *p) +{ +#ifdef MOZ_DMD + gSqliteMemoryUsed -= SqliteMallocSizeOfOnFree(p); +#endif + ::free(p); +} + +static void *sqliteMemRealloc(void *p, int n) +{ +#ifdef MOZ_DMD + gSqliteMemoryUsed -= SqliteMallocSizeOfOnFree(p); + void *pnew = ::realloc(p, n); + if (pnew) { + gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(pnew); + } else { + // realloc failed; undo the SqliteMallocSizeOfOnFree from above + gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(p); + } + return pnew; +#else + return ::realloc(p, n); +#endif +} + +static int sqliteMemSize(void *p) +{ + return ::moz_malloc_usable_size(p); +} + +static int sqliteMemRoundup(int n) +{ + n = malloc_good_size(n); + + // jemalloc can return blocks of size 2 and 4, but SQLite requires that all + // allocations be 8-aligned. So we round up sub-8 requests to 8. This + // wastes a small amount of memory but is obviously safe. + return n <= 8 ? 8 : n; +} + +static int sqliteMemInit(void *p) +{ + return 0; +} + +static void sqliteMemShutdown(void *p) +{ +} + +const sqlite3_mem_methods memMethods = { + &sqliteMemMalloc, + &sqliteMemFree, + &sqliteMemRealloc, + &sqliteMemSize, + &sqliteMemRoundup, + &sqliteMemInit, + &sqliteMemShutdown, + nullptr +}; + +} // namespace + +#endif // MOZ_STORAGE_MEMORY + +namespace mozilla { + +AutoSQLiteLifetime::AutoSQLiteLifetime() +{ + if (++AutoSQLiteLifetime::sSingletonEnforcer != 1) { + NS_RUNTIMEABORT("multiple instances of AutoSQLiteLifetime constructed!"); + } + +#ifdef MOZ_STORAGE_MEMORY + sResult = ::sqlite3_config(SQLITE_CONFIG_MALLOC, &memMethods); +#else + sResult = SQLITE_OK; +#endif + + if (sResult == SQLITE_OK) { + // TODO (bug 1191405): do not preallocate the connections caches until we + // have figured the impact on our consumers and memory. + sqlite3_config(SQLITE_CONFIG_PAGECACHE, NULL, 0, 0); + + // Explicitly initialize sqlite3. Although this is implicitly called by + // various sqlite3 functions (and the sqlite3_open calls in our case), + // the documentation suggests calling this directly. So we do. + sResult = ::sqlite3_initialize(); + } +} + +AutoSQLiteLifetime::~AutoSQLiteLifetime() +{ + // Shutdown the sqlite3 API. Warn if shutdown did not turn out okay, but + // there is nothing actionable we can do in that case. + sResult = ::sqlite3_shutdown(); + NS_WARNING_ASSERTION(sResult == SQLITE_OK, + "sqlite3 did not shutdown cleanly."); +} + +int AutoSQLiteLifetime::sSingletonEnforcer = 0; +int AutoSQLiteLifetime::sResult = SQLITE_MISUSE; + +} // namespace mozilla diff --git a/toolkit/xre/AutoSQLiteLifetime.h b/toolkit/xre/AutoSQLiteLifetime.h new file mode 100644 --- /dev/null +++ b/toolkit/xre/AutoSQLiteLifetime.h @@ -0,0 +1,24 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_AutoSQLiteLifetime_h +#define mozilla_AutoSQLiteLifetime_h + +namespace mozilla { + +class AutoSQLiteLifetime final +{ +private: + static int sSingletonEnforcer; + static int sResult; +public: + AutoSQLiteLifetime(); + ~AutoSQLiteLifetime(); + static int getInitResult() { return AutoSQLiteLifetime::sResult; } +}; + +} // namespace mozilla + +#endif