Skip creating external textures that exceeds size limit

Skia always accept an Android buffer and wraps it around with a texture
even if its size exceeds the limit GL exposes. Therefore let's skip
creating the texture in SurfaceFlinger and outputs an error log to
logcat.

I chose to do it in SurfaceFlinger rather than RenderEngine is because
the external texture mapping is designed to be asynchronous, so it'd be
better to keep that way. The limit is also exposed out of RenderEngine
so SurfaceFlinger can check it before creating external textures as
well.

Bug: 190399306
Bug: 204316511
Test: The test mentioned in the bug fails instead of crashing
SurfaceFlinger.
Test: atest SurfaceFlinger_test
Test: atest libsurfaceflinger_unittest

Change-Id: I52d253ed5a10f0e4ade372048721913405ed668a
(cherry picked from commit 9c9c19134593655c36fe70aaa45a91ad4f75e36f)
Merged-In: I52d253ed5a10f0e4ade372048721913405ed668a
(cherry picked from commit 2914a57d755051a3e5f05154d784a08019500946)
Merged-In:I52d253ed5a10f0e4ade372048721913405ed668a
diff --git a/services/surfaceflinger/BufferQueueLayer.cpp b/services/surfaceflinger/BufferQueueLayer.cpp
index 6b6d434..99e470d 100644
--- a/services/surfaceflinger/BufferQueueLayer.cpp
+++ b/services/surfaceflinger/BufferQueueLayer.cpp
@@ -515,13 +515,10 @@
 }
 
 status_t BufferQueueLayer::setDefaultBufferProperties(uint32_t w, uint32_t h, PixelFormat format) {
-    uint32_t const maxSurfaceDims =
-          std::min(mFlinger->getMaxTextureSize(), mFlinger->getMaxViewportDims());
-
     // never allow a surface larger than what our underlying GL implementation
     // can handle.
-    if ((uint32_t(w) > maxSurfaceDims) || (uint32_t(h) > maxSurfaceDims)) {
-        ALOGE("dimensions too large %u x %u", uint32_t(w), uint32_t(h));
+    if (mFlinger->exceedsMaxRenderTargetSize(w, h)) {
+        ALOGE("dimensions too large %" PRIu32 " x %" PRIu32, w, h);
         return BAD_VALUE;
     }
 
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index fe43a20..bbf6a2d 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -792,6 +792,8 @@
                                     ? renderengine::RenderEngine::ContextPriority::REALTIME
                                     : renderengine::RenderEngine::ContextPriority::MEDIUM)
                     .build()));
+    mMaxRenderTargetSize =
+            std::min(getRenderEngine().getMaxTextureSize(), getRenderEngine().getMaxViewportDims());
 
     // Set SF main policy after initializing RenderEngine which has its own policy.
     if (!SetTaskProfiles(0, {"SFMainPolicy"})) {
@@ -879,14 +881,6 @@
     }
 }
 
-size_t SurfaceFlinger::getMaxTextureSize() const {
-    return getRenderEngine().getMaxTextureSize();
-}
-
-size_t SurfaceFlinger::getMaxViewportDims() const {
-    return getRenderEngine().getMaxViewportDims();
-}
-
 // ----------------------------------------------------------------------------
 
 bool SurfaceFlinger::authenticateSurfaceTexture(
@@ -4216,17 +4210,30 @@
     }
     bool bufferChanged = what & layer_state_t::eBufferChanged;
     bool cacheIdChanged = what & layer_state_t::eCachedBufferChanged;
+    bool bufferSizeExceedsLimit = false;
     std::shared_ptr<renderengine::ExternalTexture> buffer;
     if (bufferChanged && cacheIdChanged && s.buffer != nullptr) {
-        ClientCache::getInstance().add(s.cachedBuffer, s.buffer);
-        buffer = ClientCache::getInstance().get(s.cachedBuffer);
+        bufferSizeExceedsLimit =
+                exceedsMaxRenderTargetSize(s.buffer->getWidth(), s.buffer->getHeight());
+        if (!bufferSizeExceedsLimit) {
+            ClientCache::getInstance().add(s.cachedBuffer, s.buffer);
+            buffer = ClientCache::getInstance().get(s.cachedBuffer);
+        }
     } else if (cacheIdChanged) {
         buffer = ClientCache::getInstance().get(s.cachedBuffer);
     } else if (bufferChanged && s.buffer != nullptr) {
-        buffer = std::make_shared<
-                renderengine::ExternalTexture>(s.buffer, getRenderEngine(),
-                                               renderengine::ExternalTexture::Usage::READABLE);
+        bufferSizeExceedsLimit =
+                exceedsMaxRenderTargetSize(s.buffer->getWidth(), s.buffer->getHeight());
+        if (!bufferSizeExceedsLimit) {
+            buffer = std::make_shared<
+                    renderengine::ExternalTexture>(s.buffer, getRenderEngine(),
+                                                   renderengine::ExternalTexture::Usage::READABLE);
+        }
     }
+    ALOGE_IF(bufferSizeExceedsLimit,
+             "Attempted to create an ExternalTexture for layer %s that exceeds render target size "
+             "limit.",
+             layer->getDebugName());
     if (buffer) {
         const bool frameNumberChanged = what & layer_state_t::eFrameNumberChanged;
         const uint64_t frameNumber = frameNumberChanged
@@ -6186,6 +6193,13 @@
                                              const sp<IScreenCaptureListener>& captureListener) {
     ATRACE_CALL();
 
+    if (exceedsMaxRenderTargetSize(bufferSize.getWidth(), bufferSize.getHeight())) {
+        ALOGE("Attempted to capture screen with size (%" PRId32 ", %" PRId32
+              ") that exceeds render target size limit.",
+              bufferSize.getWidth(), bufferSize.getHeight());
+        return BAD_VALUE;
+    }
+
     // Loop over all visible layers to see whether there's any protected layer. A protected layer is
     // typically a layer with DRM contents, or have the GRALLOC_USAGE_PROTECTED set on the buffer.
     // A protected layer has no implication on whether it's secure, which is explicitly set by
diff --git a/services/surfaceflinger/SurfaceFlinger.h b/services/surfaceflinger/SurfaceFlinger.h
index c3d837e..c63225f 100644
--- a/services/surfaceflinger/SurfaceFlinger.h
+++ b/services/surfaceflinger/SurfaceFlinger.h
@@ -930,8 +930,9 @@
 
     void readPersistentProperties();
 
-    size_t getMaxTextureSize() const;
-    size_t getMaxViewportDims() const;
+    bool exceedsMaxRenderTargetSize(uint32_t width, uint32_t height) const {
+        return width > mMaxRenderTargetSize || height > mMaxRenderTargetSize;
+    }
 
     int getMaxAcquiredBufferCountForCurrentRefreshRate(uid_t uid) const;
 
@@ -1378,6 +1379,9 @@
 
     SurfaceFlingerBE mBE;
     std::unique_ptr<compositionengine::CompositionEngine> mCompositionEngine;
+    // mMaxRenderTargetSize is only set once in init() so it doesn't need to be protected by
+    // any mutex.
+    size_t mMaxRenderTargetSize{1};
 
     const std::string mHwcServiceName;
 
diff --git a/services/surfaceflinger/tests/ScreenCapture_test.cpp b/services/surfaceflinger/tests/ScreenCapture_test.cpp
index 6912fcf..ab2064e 100644
--- a/services/surfaceflinger/tests/ScreenCapture_test.cpp
+++ b/services/surfaceflinger/tests/ScreenCapture_test.cpp
@@ -515,7 +515,8 @@
 }
 
 TEST_F(ScreenCaptureTest, CaptureInvalidLayer) {
-    sp<SurfaceControl> redLayer = createLayer(String8("Red surface"), 60, 60, 0);
+    sp<SurfaceControl> redLayer = createLayer(String8("Red surface"), 60, 60,
+                                              ISurfaceComposerClient::eFXSurfaceBufferState);
 
     ASSERT_NO_FATAL_FAILURE(fillBufferQueueLayerColor(redLayer, Color::RED, 60, 60));
 
@@ -532,6 +533,21 @@
     ASSERT_EQ(NAME_NOT_FOUND, ScreenCapture::captureLayers(args, captureResults));
 }
 
+TEST_F(ScreenCaptureTest, CaptureTooLargeLayer) {
+    sp<SurfaceControl> redLayer = createLayer(String8("Red surface"), 60, 60);
+    ASSERT_NO_FATAL_FAILURE(fillBufferQueueLayerColor(redLayer, Color::RED, 60, 60));
+
+    Transaction().show(redLayer).setLayer(redLayer, INT32_MAX).apply(true);
+
+    LayerCaptureArgs captureArgs;
+    captureArgs.layerHandle = redLayer->getHandle();
+    captureArgs.frameScaleX = INT32_MAX / 60;
+    captureArgs.frameScaleY = INT32_MAX / 60;
+
+    ScreenCaptureResults captureResults;
+    ASSERT_EQ(BAD_VALUE, ScreenCapture::captureLayers(captureArgs, captureResults));
+}
+
 TEST_F(ScreenCaptureTest, CaptureSecureLayer) {
     sp<SurfaceControl> redLayer = createLayer(String8("Red surface"), 60, 60,
                                               ISurfaceComposerClient::eFXSurfaceBufferState);
diff --git a/services/surfaceflinger/tests/unittests/CompositionTest.cpp b/services/surfaceflinger/tests/unittests/CompositionTest.cpp
index 560f139..52a36a2 100644
--- a/services/surfaceflinger/tests/unittests/CompositionTest.cpp
+++ b/services/surfaceflinger/tests/unittests/CompositionTest.cpp
@@ -108,6 +108,8 @@
 
         mComposer = new Hwc2::mock::Composer();
         mFlinger.setupComposer(std::unique_ptr<Hwc2::Composer>(mComposer));
+
+        mFlinger.mutableMaxRenderTargetSize() = 16384;
     }
 
     ~CompositionTest() {
@@ -519,8 +521,6 @@
 
     static void setupLatchedBuffer(CompositionTest* test, sp<BufferQueueLayer> layer) {
         // TODO: Eliminate the complexity of actually creating a buffer
-        EXPECT_CALL(*test->mRenderEngine, getMaxTextureSize()).WillOnce(Return(16384));
-        EXPECT_CALL(*test->mRenderEngine, getMaxViewportDims()).WillOnce(Return(16384));
         status_t err =
                 layer->setDefaultBufferProperties(LayerProperties::WIDTH, LayerProperties::HEIGHT,
                                                   LayerProperties::FORMAT);
diff --git a/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h b/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h
index cf67593..3802e0d 100644
--- a/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h
+++ b/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h
@@ -435,6 +435,7 @@
     auto& mutableTransactionFlags() { return mFlinger->mTransactionFlags; }
     auto& mutablePowerAdvisor() { return mFlinger->mPowerAdvisor; }
     auto& mutableDebugDisableHWC() { return mFlinger->mDebugDisableHWC; }
+    auto& mutableMaxRenderTargetSize() { return mFlinger->mMaxRenderTargetSize; }
 
     auto& mutableHwcDisplayData() { return getHwComposer().mDisplayData; }
     auto& mutableHwcPhysicalDisplayIdMap() { return getHwComposer().mPhysicalDisplayIdMap; }