From 3f7526b98d3e851df710fe02023b9deb3661d197 Mon Sep 17 00:00:00 2001 From: Kai Engert Date: Tue, 24 Oct 2017 12:09:44 +0200 Subject: Backport mozbz#730495 for rhbz#1496563 --- firefox.spec | 12 +- sqlcompat-ff57-1-backport-730495 | 524 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 535 insertions(+), 1 deletion(-) create mode 100644 sqlcompat-ff57-1-backport-730495 diff --git a/firefox.spec b/firefox.spec index d3fb372..79f272f 100644 --- a/firefox.spec +++ b/firefox.spec @@ -103,7 +103,7 @@ ExcludeArch: armv7hl Summary: Mozilla Firefox Web browser Name: firefox Version: 57.0 -Release: 0.6%{?pre_tag}%{?dist} +Release: 0.7%{?pre_tag}%{?dist} URL: https://www.mozilla.org/firefox/ License: MPLv1.1 or GPLv2+ or LGPLv2+ Group: Applications/Internet @@ -161,6 +161,9 @@ Patch412: mozilla-1337988.patch Patch413: mozilla-1353817.patch Patch416: mozilla-1399611.patch +# Better compatibility with NSS sql database format, rhbz#1496563 +Patch481: sqlcompat-ff57-1-backport-730495 + # Debian patches Patch500: mozilla-440908.patch @@ -340,6 +343,10 @@ This package contains results of tests executed during build. %patch413 -p1 -b .1353817 %patch416 -p1 -b .1399611 +%if 0%{?fedora} > 27 +%patch481 -p1 -b .sqlcompat-1 +%endif + # Debian extension patch %patch500 -p1 -b .440908 @@ -860,6 +867,9 @@ gtk-update-icon-cache %{_datadir}/icons/hicolor &>/dev/null || : #--------------------------------------------------------------------- %changelog +* Tue Oct 24 2017 Kai Engert - 57.0-0.7 +- Backport mozbz#730495 for rhbz#1496563 + * Tue Oct 24 2017 Martin Stransky - 57.0-0.6 - Updated to 57.0 Beta 11 diff --git a/sqlcompat-ff57-1-backport-730495 b/sqlcompat-ff57-1-backport-730495 new file mode 100644 index 0000000..05b0eaa --- /dev/null +++ b/sqlcompat-ff57-1-backport-730495 @@ -0,0 +1,524 @@ +# HG changeset patch +# Parent 5de7eafc3ceca2196d84d5b6106e01046efda034 + +diff --git a/security/manager/ssl/nsNSSComponent.cpp b/security/manager/ssl/nsNSSComponent.cpp +--- a/security/manager/ssl/nsNSSComponent.cpp ++++ b/security/manager/ssl/nsNSSComponent.cpp +@@ -13,7 +13,6 @@ + #include "SharedSSLState.h" + #include "cert.h" + #include "certdb.h" +-#include "mozStorageCID.h" + #include "mozilla/ArrayUtils.h" + #include "mozilla/Assertions.h" + #include "mozilla/Casting.h" +@@ -2038,14 +2037,6 @@ nsNSSComponent::Init() + return NS_ERROR_NOT_AVAILABLE; + } + +- // To avoid a sqlite3_config race in NSS init, as a workaround for +- // bug 730495, we require the storage service to get initialized first. +- nsCOMPtr storageService = +- do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID); +- if (!storageService) { +- return NS_ERROR_NOT_AVAILABLE; +- } +- + MOZ_LOG(gPIPNSSLog, LogLevel::Debug, ("Beginning NSS initialization\n")); + + nsresult rv = InitializePIPNSSBundle(); +diff --git a/storage/TelemetryVFS.cpp b/storage/TelemetryVFS.cpp +--- a/storage/TelemetryVFS.cpp ++++ b/storage/TelemetryVFS.cpp +@@ -834,6 +834,11 @@ xNextSystemCall(sqlite3_vfs *vfs, const + namespace mozilla { + namespace storage { + ++const char *GetVFSName() ++{ ++ return "telemetry-vfs"; ++} ++ + sqlite3_vfs* ConstructTelemetryVFS() + { + #if defined(XP_WIN) +@@ -867,7 +872,7 @@ sqlite3_vfs* ConstructTelemetryVFS() + MOZ_ASSERT(vfs->iVersion <= LAST_KNOWN_VFS_VERSION); + tvfs->szOsFile = sizeof(telemetry_file) - sizeof(sqlite3_file) + vfs->szOsFile; + tvfs->mxPathname = vfs->mxPathname; +- tvfs->zName = "telemetry-vfs"; ++ tvfs->zName = GetVFSName(); + tvfs->pAppData = vfs; + tvfs->xOpen = xOpen; + tvfs->xDelete = xDelete; +diff --git a/storage/mozStorageConnection.cpp b/storage/mozStorageConnection.cpp +--- a/storage/mozStorageConnection.cpp ++++ b/storage/mozStorageConnection.cpp +@@ -73,6 +73,8 @@ namespace storage { + + using mozilla::dom::quota::QuotaObject; + ++const char *GetVFSName(); ++ + namespace { + + int +@@ -627,7 +629,7 @@ Connection::initialize() + AUTO_PROFILER_LABEL("Connection::initialize", STORAGE); + + // in memory database requested, sqlite uses a magic file name +- int srv = ::sqlite3_open_v2(":memory:", &mDBConn, mFlags, nullptr); ++ int srv = ::sqlite3_open_v2(":memory:", &mDBConn, mFlags, GetVFSName()); + if (srv != SQLITE_OK) { + mDBConn = nullptr; + return convertResultCode(srv); +@@ -660,7 +662,7 @@ Connection::initialize(nsIFile *aDatabas + #else + static const char* sIgnoreLockingVFS = "unix-none"; + #endif +- const char* vfs = mIgnoreLockingMode ? sIgnoreLockingVFS : nullptr; ++ const char* vfs = mIgnoreLockingMode ? sIgnoreLockingVFS : GetVFSName(); + + int srv = ::sqlite3_open_v2(NS_ConvertUTF16toUTF8(path).get(), &mDBConn, + mFlags, vfs); +@@ -694,7 +696,7 @@ Connection::initialize(nsIFileURL *aFile + rv = aFileURL->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + +- int srv = ::sqlite3_open_v2(spec.get(), &mDBConn, mFlags, nullptr); ++ int srv = ::sqlite3_open_v2(spec.get(), &mDBConn, mFlags, GetVFSName()); + if (srv != SQLITE_OK) { + mDBConn = nullptr; + return convertResultCode(srv); +diff --git a/storage/mozStorageService.cpp b/storage/mozStorageService.cpp +--- a/storage/mozStorageService.cpp ++++ b/storage/mozStorageService.cpp +@@ -24,6 +24,7 @@ + #include "mozIStoragePendingStatement.h" + + #include "sqlite3.h" ++#include "mozilla/AutoSQLiteLifetime.h" + + #ifdef SQLITE_OS_WIN + // "windows.h" was included and it can #define lots of things we care about... +@@ -32,13 +33,6 @@ + + #include "nsIPromptService.h" + +-#ifdef MOZ_STORAGE_MEMORY +-# include "mozmemory.h" +-# ifdef MOZ_DMD +-# include "DMD.h" +-# endif +-#endif +- + //////////////////////////////////////////////////////////////////////////////// + //// Defines + +@@ -282,12 +276,6 @@ Service::~Service() + if (rc != SQLITE_OK) + NS_WARNING("Failed to unregister sqlite vfs wrapper."); + +- // Shutdown the sqlite3 API. Warn if shutdown did not turn out okay, but +- // there is nothing actionable we can do in that case. +- rc = ::sqlite3_shutdown(); +- if (rc != SQLITE_OK) +- NS_WARNING("sqlite3 did not shutdown cleanly."); +- + shutdown(); // To release sXPConnect. + + gService = nullptr; +@@ -400,121 +388,7 @@ Service::shutdown() + } + + sqlite3_vfs *ConstructTelemetryVFS(); +- +-#ifdef MOZ_STORAGE_MEMORY +- +-namespace { +- +-// By default, SQLite tracks the size of all its heap blocks by adding an extra +-// 8 bytes at the start of the block to hold the size. Unfortunately, this +-// causes a lot of 2^N-sized allocations to be rounded up by jemalloc +-// allocator, wasting memory. For example, a request for 1024 bytes has 8 +-// bytes added, becoming a request for 1032 bytes, and jemalloc rounds this up +-// to 2048 bytes, wasting 1012 bytes. (See bug 676189 for more details.) +-// +-// So we register jemalloc as the malloc implementation, which avoids this +-// 8-byte overhead, and thus a lot of waste. This requires us to provide a +-// function, sqliteMemRoundup(), which computes the actual size that will be +-// allocated for a given request. SQLite uses this function before all +-// allocations, and may be able to use any excess bytes caused by the rounding. +-// +-// Note: the wrappers for malloc, realloc and moz_malloc_usable_size are +-// necessary because the sqlite_mem_methods type signatures differ slightly +-// from the standard ones -- they use int instead of size_t. But we don't need +-// a wrapper for free. +- +-#ifdef MOZ_DMD +- +-// sqlite does its own memory accounting, and we use its numbers in our memory +-// reporters. But we don't want sqlite's heap blocks to show up in DMD's +-// output as unreported, so we mark them as reported when they're allocated and +-// mark them as unreported when they are freed. +-// +-// In other words, we are marking all sqlite heap blocks as reported even +-// though we're not reporting them ourselves. Instead we're trusting that +-// sqlite is fully and correctly accounting for all of its heap blocks via its +-// own memory accounting. Well, we don't have to trust it entirely, because +-// it's easy to keep track (while doing this DMD-specific marking) of exactly +-// how much memory SQLite is using. And we can compare that against what +-// SQLite reports it is using. +- +-MOZ_DEFINE_MALLOC_SIZE_OF_ON_ALLOC(SqliteMallocSizeOfOnAlloc) +-MOZ_DEFINE_MALLOC_SIZE_OF_ON_FREE(SqliteMallocSizeOfOnFree) +- +-#endif +- +-static void *sqliteMemMalloc(int n) +-{ +- void* p = ::malloc(n); +-#ifdef MOZ_DMD +- gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(p); +-#endif +- return p; +-} +- +-static void sqliteMemFree(void *p) +-{ +-#ifdef MOZ_DMD +- gSqliteMemoryUsed -= SqliteMallocSizeOfOnFree(p); +-#endif +- ::free(p); +-} +- +-static void *sqliteMemRealloc(void *p, int n) +-{ +-#ifdef MOZ_DMD +- gSqliteMemoryUsed -= SqliteMallocSizeOfOnFree(p); +- void *pnew = ::realloc(p, n); +- if (pnew) { +- gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(pnew); +- } else { +- // realloc failed; undo the SqliteMallocSizeOfOnFree from above +- gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(p); +- } +- return pnew; +-#else +- return ::realloc(p, n); +-#endif +-} +- +-static int sqliteMemSize(void *p) +-{ +- return ::moz_malloc_usable_size(p); +-} +- +-static int sqliteMemRoundup(int n) +-{ +- n = malloc_good_size(n); +- +- // jemalloc can return blocks of size 2 and 4, but SQLite requires that all +- // allocations be 8-aligned. So we round up sub-8 requests to 8. This +- // wastes a small amount of memory but is obviously safe. +- return n <= 8 ? 8 : n; +-} +- +-static int sqliteMemInit(void *p) +-{ +- return 0; +-} +- +-static void sqliteMemShutdown(void *p) +-{ +-} +- +-const sqlite3_mem_methods memMethods = { +- &sqliteMemMalloc, +- &sqliteMemFree, +- &sqliteMemRealloc, +- &sqliteMemSize, +- &sqliteMemRoundup, +- &sqliteMemInit, +- &sqliteMemShutdown, +- nullptr +-}; +- +-} // namespace +- +-#endif // MOZ_STORAGE_MEMORY ++const char *GetVFSName(); + + static const char* sObserverTopics[] = { + "memory-pressure", +@@ -527,28 +401,13 @@ Service::initialize() + { + MOZ_ASSERT(NS_IsMainThread(), "Must be initialized on the main thread"); + +- int rc; +- +-#ifdef MOZ_STORAGE_MEMORY +- rc = ::sqlite3_config(SQLITE_CONFIG_MALLOC, &memMethods); +- if (rc != SQLITE_OK) +- return convertResultCode(rc); +-#endif +- +- // TODO (bug 1191405): do not preallocate the connections caches until we +- // have figured the impact on our consumers and memory. +- sqlite3_config(SQLITE_CONFIG_PAGECACHE, NULL, 0, 0); +- +- // Explicitly initialize sqlite3. Although this is implicitly called by +- // various sqlite3 functions (and the sqlite3_open calls in our case), +- // the documentation suggests calling this directly. So we do. +- rc = ::sqlite3_initialize(); ++ int rc = AutoSQLiteLifetime::getInitResult(); + if (rc != SQLITE_OK) + return convertResultCode(rc); + + mSqliteVFS = ConstructTelemetryVFS(); + if (mSqliteVFS) { +- rc = sqlite3_vfs_register(mSqliteVFS, 1); ++ rc = sqlite3_vfs_register(mSqliteVFS, 0); + if (rc != SQLITE_OK) + return convertResultCode(rc); + } else { +diff --git a/toolkit/xre/Bootstrap.cpp b/toolkit/xre/Bootstrap.cpp +--- a/toolkit/xre/Bootstrap.cpp ++++ b/toolkit/xre/Bootstrap.cpp +@@ -6,11 +6,15 @@ + #include "mozilla/Bootstrap.h" + #include "nsXPCOM.h" + ++#include "AutoSQLiteLifetime.h" ++ + namespace mozilla { + + class BootstrapImpl final : public Bootstrap + { + protected: ++ AutoSQLiteLifetime mSQLLT; ++ + virtual void Dispose() override + { + delete this; +diff --git a/toolkit/xre/moz.build b/toolkit/xre/moz.build +--- a/toolkit/xre/moz.build ++++ b/toolkit/xre/moz.build +@@ -30,7 +30,7 @@ EXPORTS += [ + 'nsIAppStartupNotifier.h', + ] + +-EXPORTS.mozilla += ['Bootstrap.h'] ++EXPORTS.mozilla += ['AutoSQLiteLifetime.h', 'Bootstrap.h'] + + if CONFIG['MOZ_INSTRUMENT_EVENT_LOOP']: + EXPORTS += ['EventTracer.h'] +@@ -91,6 +91,7 @@ if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'andr + ] + + UNIFIED_SOURCES += [ ++ 'AutoSQLiteLifetime.cpp', + 'Bootstrap.cpp', + 'CreateAppData.cpp', + 'nsAppStartupNotifier.cpp', +diff --git a/toolkit/xre/AutoSQLiteLifetime.cpp b/toolkit/xre/AutoSQLiteLifetime.cpp +new file mode 100644 +--- /dev/null ++++ b/toolkit/xre/AutoSQLiteLifetime.cpp +@@ -0,0 +1,167 @@ ++/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ ++/* This Source Code Form is subject to the terms of the Mozilla Public ++ * License, v. 2.0. If a copy of the MPL was not distributed with this ++ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ ++ ++#include "nsDebug.h" ++#include "AutoSQLiteLifetime.h" ++#include "sqlite3.h" ++ ++#ifdef MOZ_STORAGE_MEMORY ++# include "mozmemory.h" ++# ifdef MOZ_DMD ++# include "DMD.h" ++# endif ++ ++namespace { ++ ++// By default, SQLite tracks the size of all its heap blocks by adding an extra ++// 8 bytes at the start of the block to hold the size. Unfortunately, this ++// causes a lot of 2^N-sized allocations to be rounded up by jemalloc ++// allocator, wasting memory. For example, a request for 1024 bytes has 8 ++// bytes added, becoming a request for 1032 bytes, and jemalloc rounds this up ++// to 2048 bytes, wasting 1012 bytes. (See bug 676189 for more details.) ++// ++// So we register jemalloc as the malloc implementation, which avoids this ++// 8-byte overhead, and thus a lot of waste. This requires us to provide a ++// function, sqliteMemRoundup(), which computes the actual size that will be ++// allocated for a given request. SQLite uses this function before all ++// allocations, and may be able to use any excess bytes caused by the rounding. ++// ++// Note: the wrappers for malloc, realloc and moz_malloc_usable_size are ++// necessary because the sqlite_mem_methods type signatures differ slightly ++// from the standard ones -- they use int instead of size_t. But we don't need ++// a wrapper for free. ++ ++#ifdef MOZ_DMD ++ ++// sqlite does its own memory accounting, and we use its numbers in our memory ++// reporters. But we don't want sqlite's heap blocks to show up in DMD's ++// output as unreported, so we mark them as reported when they're allocated and ++// mark them as unreported when they are freed. ++// ++// In other words, we are marking all sqlite heap blocks as reported even ++// though we're not reporting them ourselves. Instead we're trusting that ++// sqlite is fully and correctly accounting for all of its heap blocks via its ++// own memory accounting. Well, we don't have to trust it entirely, because ++// it's easy to keep track (while doing this DMD-specific marking) of exactly ++// how much memory SQLite is using. And we can compare that against what ++// SQLite reports it is using. ++ ++MOZ_DEFINE_MALLOC_SIZE_OF_ON_ALLOC(SqliteMallocSizeOfOnAlloc) ++MOZ_DEFINE_MALLOC_SIZE_OF_ON_FREE(SqliteMallocSizeOfOnFree) ++ ++#endif ++ ++static void *sqliteMemMalloc(int n) ++{ ++ void* p = ::malloc(n); ++#ifdef MOZ_DMD ++ gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(p); ++#endif ++ return p; ++} ++ ++static void sqliteMemFree(void *p) ++{ ++#ifdef MOZ_DMD ++ gSqliteMemoryUsed -= SqliteMallocSizeOfOnFree(p); ++#endif ++ ::free(p); ++} ++ ++static void *sqliteMemRealloc(void *p, int n) ++{ ++#ifdef MOZ_DMD ++ gSqliteMemoryUsed -= SqliteMallocSizeOfOnFree(p); ++ void *pnew = ::realloc(p, n); ++ if (pnew) { ++ gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(pnew); ++ } else { ++ // realloc failed; undo the SqliteMallocSizeOfOnFree from above ++ gSqliteMemoryUsed += SqliteMallocSizeOfOnAlloc(p); ++ } ++ return pnew; ++#else ++ return ::realloc(p, n); ++#endif ++} ++ ++static int sqliteMemSize(void *p) ++{ ++ return ::moz_malloc_usable_size(p); ++} ++ ++static int sqliteMemRoundup(int n) ++{ ++ n = malloc_good_size(n); ++ ++ // jemalloc can return blocks of size 2 and 4, but SQLite requires that all ++ // allocations be 8-aligned. So we round up sub-8 requests to 8. This ++ // wastes a small amount of memory but is obviously safe. ++ return n <= 8 ? 8 : n; ++} ++ ++static int sqliteMemInit(void *p) ++{ ++ return 0; ++} ++ ++static void sqliteMemShutdown(void *p) ++{ ++} ++ ++const sqlite3_mem_methods memMethods = { ++ &sqliteMemMalloc, ++ &sqliteMemFree, ++ &sqliteMemRealloc, ++ &sqliteMemSize, ++ &sqliteMemRoundup, ++ &sqliteMemInit, ++ &sqliteMemShutdown, ++ nullptr ++}; ++ ++} // namespace ++ ++#endif // MOZ_STORAGE_MEMORY ++ ++namespace mozilla { ++ ++AutoSQLiteLifetime::AutoSQLiteLifetime() ++{ ++ if (++AutoSQLiteLifetime::sSingletonEnforcer != 1) { ++ NS_RUNTIMEABORT("multiple instances of AutoSQLiteLifetime constructed!"); ++ } ++ ++#ifdef MOZ_STORAGE_MEMORY ++ sResult = ::sqlite3_config(SQLITE_CONFIG_MALLOC, &memMethods); ++#else ++ sResult = SQLITE_OK; ++#endif ++ ++ if (sResult == SQLITE_OK) { ++ // TODO (bug 1191405): do not preallocate the connections caches until we ++ // have figured the impact on our consumers and memory. ++ sqlite3_config(SQLITE_CONFIG_PAGECACHE, NULL, 0, 0); ++ ++ // Explicitly initialize sqlite3. Although this is implicitly called by ++ // various sqlite3 functions (and the sqlite3_open calls in our case), ++ // the documentation suggests calling this directly. So we do. ++ sResult = ::sqlite3_initialize(); ++ } ++} ++ ++AutoSQLiteLifetime::~AutoSQLiteLifetime() ++{ ++ // Shutdown the sqlite3 API. Warn if shutdown did not turn out okay, but ++ // there is nothing actionable we can do in that case. ++ sResult = ::sqlite3_shutdown(); ++ NS_WARNING_ASSERTION(sResult == SQLITE_OK, ++ "sqlite3 did not shutdown cleanly."); ++} ++ ++int AutoSQLiteLifetime::sSingletonEnforcer = 0; ++int AutoSQLiteLifetime::sResult = SQLITE_MISUSE; ++ ++} // namespace mozilla +diff --git a/toolkit/xre/AutoSQLiteLifetime.h b/toolkit/xre/AutoSQLiteLifetime.h +new file mode 100644 +--- /dev/null ++++ b/toolkit/xre/AutoSQLiteLifetime.h +@@ -0,0 +1,24 @@ ++/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ ++/* This Source Code Form is subject to the terms of the Mozilla Public ++ * License, v. 2.0. If a copy of the MPL was not distributed with this ++ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ ++ ++#ifndef mozilla_AutoSQLiteLifetime_h ++#define mozilla_AutoSQLiteLifetime_h ++ ++namespace mozilla { ++ ++class AutoSQLiteLifetime final ++{ ++private: ++ static int sSingletonEnforcer; ++ static int sResult; ++public: ++ AutoSQLiteLifetime(); ++ ~AutoSQLiteLifetime(); ++ static int getInitResult() { return AutoSQLiteLifetime::sResult; } ++}; ++ ++} // namespace mozilla ++ ++#endif -- cgit