Added fix for mozbz#1673313

1 files changed, 351 insertions, 0 deletions

diff --git a/mozilla-1673313.patch b/mozilla-1673313.patch
new file mode 100644
index 0000000..549243b
--- /dev/null
+++ b/mozilla-1673313.patch
@@ -0,0 +1,351 @@
+changeset:   556172:143b4ca96ec9
+tag:         tip
+parent:      556169:61c35792ca70
+user:        stransky <stransky@redhat.com>
+date:        Mon Oct 26 12:15:49 2020 +0100
+files:       widget/gtk/WindowSurfaceWayland.cpp widget/gtk/WindowSurfaceWayland.h
+description:
+Bug 1673313 [Wayland] Don't fail when Shm allocation fails, r?jhorak
+
+- Make WaylandAllocateShmMemory() fallible.
+- Implement WaylandReAllocateShmMemory() to re-allocate Shm pool.
+- Remove WaylandShmPool::Resize() and use WaylandShmPool::Create() only.
+- Implement and use WaylandShmPool::Release().
+- Make WindowSurfaceWayland::CreateWaylandBuffer() as fallible.
+
+Differential Revision: https://phabricator.services.mozilla.com/D94735
+
+
+diff --git a/widget/gtk/WindowSurfaceWayland.cpp b/widget/gtk/WindowSurfaceWayland.cpp
+--- a/widget/gtk/WindowSurfaceWayland.cpp
++++ b/widget/gtk/WindowSurfaceWayland.cpp
+@@ -209,14 +209,23 @@ RefPtr<nsWaylandDisplay> WindowBackBuffe
+ }
+ 
+ static int WaylandAllocateShmMemory(int aSize) {
+-  static int counter = 0;
+-  nsPrintfCString shmName("/wayland.mozilla.ipc.%d", counter++);
+-  int fd = shm_open(shmName.get(), O_CREAT | O_RDWR | O_EXCL, 0600);
+-  if (fd >= 0) {
+-    shm_unlink(shmName.get());
+-  } else {
+-    printf_stderr("Unable to SHM memory segment\n");
+-    MOZ_CRASH();
++  int fd = -1;
++  do {
++    static int counter = 0;
++    nsPrintfCString shmName("/wayland.mozilla.ipc.%d", counter++);
++    fd = shm_open(shmName.get(), O_CREAT | O_RDWR | O_EXCL, 0600);
++    if (fd >= 0) {
++      // We don't want to use leaked file
++      if (shm_unlink(shmName.get()) != 0) {
++        NS_WARNING("shm_unlink failed");
++        return -1;
++      }
++    }
++  } while (fd < 0 && errno == EEXIST);
++
++  if (fd < 0) {
++    NS_WARNING(nsPrintfCString("shm_open failed: %s", strerror(errno)).get());
++    return -1;
+   }
+ 
+   int ret = 0;
+@@ -225,59 +234,103 @@ static int WaylandAllocateShmMemory(int 
+     ret = posix_fallocate(fd, 0, aSize);
+   } while (ret == EINTR);
+   if (ret != 0) {
++    NS_WARNING(
++        nsPrintfCString("posix_fallocate() fails to allocate shm memory: %s",
++                        strerror(ret))
++            .get());
+     close(fd);
+-    MOZ_CRASH("posix_fallocate() fails to allocate shm memory");
++    return -1;
+   }
+ #else
+   do {
+     ret = ftruncate(fd, aSize);
+   } while (ret < 0 && errno == EINTR);
+   if (ret < 0) {
++    NS_WARNING(nsPrintfCString("ftruncate() fails to allocate shm memory: %s",
++                               strerror(ret))
++                   .get());
+     close(fd);
+-    MOZ_CRASH("ftruncate() fails to allocate shm memory");
++    fd = -1;
+   }
+ #endif
+ 
+   return fd;
+ }
+ 
+-WaylandShmPool::WaylandShmPool(RefPtr<nsWaylandDisplay> aWaylandDisplay,
+-                               int aSize)
+-    : mAllocatedSize(aSize) {
+-  mShmPoolFd = WaylandAllocateShmMemory(mAllocatedSize);
+-  mImageData = mmap(nullptr, mAllocatedSize, PROT_READ | PROT_WRITE, MAP_SHARED,
+-                    mShmPoolFd, 0);
+-  MOZ_RELEASE_ASSERT(mImageData != MAP_FAILED,
+-                     "Unable to map drawing surface!");
++static bool WaylandReAllocateShmMemory(int aFd, int aSize) {
++  if (ftruncate(aFd, aSize) < 0) {
++    return false;
++  }
++#ifdef HAVE_POSIX_FALLOCATE
++  do {
++    errno = posix_fallocate(aFd, 0, aSize);
++  } while (errno == EINTR);
++  if (errno != 0) {
++    return false;
++  }
++#endif
++  return true;
++}
+ 
+-  mShmPool =
+-      wl_shm_create_pool(aWaylandDisplay->GetShm(), mShmPoolFd, mAllocatedSize);
++WaylandShmPool::WaylandShmPool()
++    : mShmPool(nullptr),
++      mShmPoolFd(-1),
++      mAllocatedSize(0),
++      mImageData(MAP_FAILED){};
+ 
+-  // We set our queue to get mShmPool events at compositor thread.
+-  wl_proxy_set_queue((struct wl_proxy*)mShmPool,
+-                     aWaylandDisplay->GetEventQueue());
++void WaylandShmPool::Release() {
++  if (mImageData != MAP_FAILED) {
++    munmap(mImageData, mAllocatedSize);
++    mImageData = MAP_FAILED;
++  }
++  if (mShmPool) {
++    wl_shm_pool_destroy(mShmPool);
++    mShmPool = 0;
++  }
++  if (mShmPoolFd >= 0) {
++    close(mShmPoolFd);
++    mShmPoolFd = -1;
++  }
+ }
+ 
+-bool WaylandShmPool::Resize(int aSize) {
++bool WaylandShmPool::Create(RefPtr<nsWaylandDisplay> aWaylandDisplay,
++                            int aSize) {
+   // We do size increase only
+-  if (aSize <= mAllocatedSize) return true;
+-
+-  if (ftruncate(mShmPoolFd, aSize) < 0) return false;
++  if (aSize <= mAllocatedSize) {
++    return true;
++  }
+ 
+-#ifdef HAVE_POSIX_FALLOCATE
+-  do {
+-    errno = posix_fallocate(mShmPoolFd, 0, aSize);
+-  } while (errno == EINTR);
+-  if (errno != 0) return false;
+-#endif
++  if (mShmPoolFd < 0) {
++    mShmPoolFd = WaylandAllocateShmMemory(aSize);
++    if (mShmPoolFd < 0) {
++      return false;
++    }
++  } else {
++    if (!WaylandReAllocateShmMemory(mShmPoolFd, aSize)) {
++      Release();
++      return false;
++    }
++  }
+ 
+-  wl_shm_pool_resize(mShmPool, aSize);
+-
+-  munmap(mImageData, mAllocatedSize);
+-
++  if (mImageData != MAP_FAILED) {
++    munmap(mImageData, mAllocatedSize);
++  }
+   mImageData =
+       mmap(nullptr, aSize, PROT_READ | PROT_WRITE, MAP_SHARED, mShmPoolFd, 0);
+-  if (mImageData == MAP_FAILED) return false;
++  if (mImageData == MAP_FAILED) {
++    NS_WARNING("Unable to map drawing surface!");
++    Release();
++    return false;
++  }
++
++  if (mShmPool) {
++    wl_shm_pool_resize(mShmPool, aSize);
++  } else {
++    mShmPool = wl_shm_create_pool(aWaylandDisplay->GetShm(), mShmPoolFd, aSize);
++    // We set our queue to get mShmPool events at compositor thread.
++    wl_proxy_set_queue((struct wl_proxy*)mShmPool,
++                       aWaylandDisplay->GetEventQueue());
++  }
+ 
+   mAllocatedSize = aSize;
+   return true;
+@@ -289,11 +342,7 @@ void WaylandShmPool::SetImageDataFromPoo
+   memcpy(mImageData, aSourcePool->GetImageData(), aImageDataSize);
+ }
+ 
+-WaylandShmPool::~WaylandShmPool() {
+-  munmap(mImageData, mAllocatedSize);
+-  wl_shm_pool_destroy(mShmPool);
+-  close(mShmPoolFd);
+-}
++WaylandShmPool::~WaylandShmPool() { Release(); }
+ 
+ static void buffer_release(void* data, wl_buffer* buffer) {
+   auto surface = reinterpret_cast<WindowBackBuffer*>(data);
+@@ -302,14 +351,14 @@ static void buffer_release(void* data, w
+ 
+ static const struct wl_buffer_listener buffer_listener = {buffer_release};
+ 
+-void WindowBackBufferShm::Create(int aWidth, int aHeight) {
++bool WindowBackBufferShm::Create(int aWidth, int aHeight) {
+   MOZ_ASSERT(!IsAttached(), "We can't create attached buffers.");
+-  MOZ_ASSERT(!mWLBuffer, "there is wl_buffer already!");
+ 
+-  int newBufferSize = aWidth * aHeight * BUFFER_BPP;
+-  if (!mShmPool.Resize(newBufferSize)) {
+-    mWLBuffer = nullptr;
+-    return;
++  ReleaseShmSurface();
++
++  int size = aWidth * aHeight * BUFFER_BPP;
++  if (!mShmPool.Create(GetWaylandDisplay(), size)) {
++    return false;
+   }
+ 
+   mWLBuffer =
+@@ -325,14 +374,16 @@ void WindowBackBufferShm::Create(int aWi
+   LOGWAYLAND(("WindowBackBufferShm::Create [%p] wl_buffer %p ID %d\n",
+               (void*)this, (void*)mWLBuffer,
+               mWLBuffer ? wl_proxy_get_id((struct wl_proxy*)mWLBuffer) : -1));
++  return true;
+ }
+ 
+ void WindowBackBufferShm::ReleaseShmSurface() {
+   LOGWAYLAND(("WindowBackBufferShm::Release [%p]\n", (void*)this));
+-
+-  wl_buffer_destroy(mWLBuffer);
++  if (mWLBuffer) {
++    wl_buffer_destroy(mWLBuffer);
++    mWLBuffer = nullptr;
++  }
+   mWidth = mHeight = 0;
+-  mWLBuffer = nullptr;
+ }
+ 
+ void WindowBackBufferShm::Clear() {
+@@ -340,16 +391,13 @@ void WindowBackBufferShm::Clear() {
+ }
+ 
+ WindowBackBufferShm::WindowBackBufferShm(
+-    WindowSurfaceWayland* aWindowSurfaceWayland, int aWidth, int aHeight)
++    WindowSurfaceWayland* aWindowSurfaceWayland)
+     : WindowBackBuffer(aWindowSurfaceWayland),
+-      mShmPool(aWindowSurfaceWayland->GetWaylandDisplay(),
+-               aWidth * aHeight * BUFFER_BPP),
++      mShmPool(),
+       mWLBuffer(nullptr),
+-      mWidth(aWidth),
+-      mHeight(aHeight),
+-      mAttached(false) {
+-  Create(aWidth, aHeight);
+-}
++      mWidth(0),
++      mHeight(0),
++      mAttached(false) {}
+ 
+ WindowBackBufferShm::~WindowBackBufferShm() { ReleaseShmSurface(); }
+ 
+@@ -357,13 +405,9 @@ bool WindowBackBufferShm::Resize(int aWi
+   if (aWidth == mWidth && aHeight == mHeight) {
+     return true;
+   }
+-
+   LOGWAYLAND(("WindowBackBufferShm::Resize [%p] %d %d\n", (void*)this, aWidth,
+               aHeight));
+-
+-  ReleaseShmSurface();
+   Create(aWidth, aHeight);
+-
+   return (mWLBuffer != nullptr);
+ }
+ 
+@@ -488,11 +532,13 @@ WindowBackBuffer* WindowSurfaceWayland::
+     return nullptr;
+   }
+ 
+-  WindowBackBuffer* buffer = new WindowBackBufferShm(this, aWidth, aHeight);
+-  if (buffer) {
+-    mShmBackupBuffer[availableBuffer] = buffer;
++  WindowBackBuffer* buffer = new WindowBackBufferShm(this);
++  if (!buffer->Create(aWidth, aHeight)) {
++    delete buffer;
++    return nullptr;
+   }
+ 
++  mShmBackupBuffer[availableBuffer] = buffer;
+   return buffer;
+ }
+ 
+diff --git a/widget/gtk/WindowSurfaceWayland.h b/widget/gtk/WindowSurfaceWayland.h
+--- a/widget/gtk/WindowSurfaceWayland.h
++++ b/widget/gtk/WindowSurfaceWayland.h
+@@ -25,14 +25,14 @@ class WindowSurfaceWayland;
+ // Allocates and owns shared memory for Wayland drawing surface
+ class WaylandShmPool {
+  public:
+-  WaylandShmPool(RefPtr<nsWaylandDisplay> aDisplay, int aSize);
+-  ~WaylandShmPool();
+-
+-  bool Resize(int aSize);
++  bool Create(RefPtr<nsWaylandDisplay> aWaylandDisplay, int aSize);
++  void Release();
+   wl_shm_pool* GetShmPool() { return mShmPool; };
+   void* GetImageData() { return mImageData; };
+   void SetImageDataFromPool(class WaylandShmPool* aSourcePool,
+                             int aImageDataSize);
++  WaylandShmPool();
++  ~WaylandShmPool();
+ 
+  private:
+   wl_shm_pool* mShmPool;
+@@ -53,6 +53,7 @@ class WindowBackBuffer {
+   virtual bool IsAttached() = 0;
+ 
+   virtual void Clear() = 0;
++  virtual bool Create(int aWidth, int aHeight) = 0;
+   virtual bool Resize(int aWidth, int aHeight) = 0;
+ 
+   virtual int GetWidth() = 0;
+@@ -87,8 +88,7 @@ class WindowBackBuffer {
+ 
+ class WindowBackBufferShm : public WindowBackBuffer {
+  public:
+-  WindowBackBufferShm(WindowSurfaceWayland* aWindowSurfaceWayland, int aWidth,
+-                      int aHeight);
++  WindowBackBufferShm(WindowSurfaceWayland* aWindowSurfaceWayland);
+   ~WindowBackBufferShm();
+ 
+   already_AddRefed<gfx::DrawTarget> Lock();
+@@ -100,6 +100,7 @@ class WindowBackBufferShm : public Windo
+   void SetAttached() { mAttached = true; };
+ 
+   void Clear();
++  bool Create(int aWidth, int aHeight);
+   bool Resize(int aWidth, int aHeight);
+   bool SetImageDataFromBuffer(class WindowBackBuffer* aSourceBuffer);
+ 
+@@ -109,7 +110,6 @@ class WindowBackBufferShm : public Windo
+   wl_buffer* GetWlBuffer() { return mWLBuffer; };
+ 
+  private:
+-  void Create(int aWidth, int aHeight);
+   void ReleaseShmSurface();
+ 
+   // WaylandShmPool provides actual shared memory we draw into
+