Merge cherrypicks of [16614247] into sc-platform-release.

Change-Id: I4980fe9995496cb0d5a5501135bf83dc9a009d9e
diff --git a/services/surfaceflinger/BufferQueueLayer.cpp b/services/surfaceflinger/BufferQueueLayer.cpp
index 6b6d434..99e470d 100644
--- a/services/surfaceflinger/BufferQueueLayer.cpp
+++ b/services/surfaceflinger/BufferQueueLayer.cpp
@@ -515,13 +515,10 @@
 }
 
 status_t BufferQueueLayer::setDefaultBufferProperties(uint32_t w, uint32_t h, PixelFormat format) {
-    uint32_t const maxSurfaceDims =
-          std::min(mFlinger->getMaxTextureSize(), mFlinger->getMaxViewportDims());
-
     // never allow a surface larger than what our underlying GL implementation
     // can handle.
-    if ((uint32_t(w) > maxSurfaceDims) || (uint32_t(h) > maxSurfaceDims)) {
-        ALOGE("dimensions too large %u x %u", uint32_t(w), uint32_t(h));
+    if (mFlinger->exceedsMaxRenderTargetSize(w, h)) {
+        ALOGE("dimensions too large %" PRIu32 " x %" PRIu32, w, h);
         return BAD_VALUE;
     }
 
diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp
index fe43a20..bbf6a2d 100644
--- a/services/surfaceflinger/SurfaceFlinger.cpp
+++ b/services/surfaceflinger/SurfaceFlinger.cpp
@@ -792,6 +792,8 @@
                                     ? renderengine::RenderEngine::ContextPriority::REALTIME
                                     : renderengine::RenderEngine::ContextPriority::MEDIUM)
                     .build()));
+    mMaxRenderTargetSize =
+            std::min(getRenderEngine().getMaxTextureSize(), getRenderEngine().getMaxViewportDims());
 
     // Set SF main policy after initializing RenderEngine which has its own policy.
     if (!SetTaskProfiles(0, {"SFMainPolicy"})) {
@@ -879,14 +881,6 @@
     }
 }
 
-size_t SurfaceFlinger::getMaxTextureSize() const {
-    return getRenderEngine().getMaxTextureSize();
-}
-
-size_t SurfaceFlinger::getMaxViewportDims() const {
-    return getRenderEngine().getMaxViewportDims();
-}
-
 // ----------------------------------------------------------------------------
 
 bool SurfaceFlinger::authenticateSurfaceTexture(
@@ -4216,17 +4210,30 @@
     }
     bool bufferChanged = what & layer_state_t::eBufferChanged;
     bool cacheIdChanged = what & layer_state_t::eCachedBufferChanged;
+    bool bufferSizeExceedsLimit = false;
     std::shared_ptr<renderengine::ExternalTexture> buffer;
     if (bufferChanged && cacheIdChanged && s.buffer != nullptr) {
-        ClientCache::getInstance().add(s.cachedBuffer, s.buffer);
-        buffer = ClientCache::getInstance().get(s.cachedBuffer);
+        bufferSizeExceedsLimit =
+                exceedsMaxRenderTargetSize(s.buffer->getWidth(), s.buffer->getHeight());
+        if (!bufferSizeExceedsLimit) {
+            ClientCache::getInstance().add(s.cachedBuffer, s.buffer);
+            buffer = ClientCache::getInstance().get(s.cachedBuffer);
+        }
     } else if (cacheIdChanged) {
         buffer = ClientCache::getInstance().get(s.cachedBuffer);
     } else if (bufferChanged && s.buffer != nullptr) {
-        buffer = std::make_shared<
-                renderengine::ExternalTexture>(s.buffer, getRenderEngine(),
-                                               renderengine::ExternalTexture::Usage::READABLE);
+        bufferSizeExceedsLimit =
+                exceedsMaxRenderTargetSize(s.buffer->getWidth(), s.buffer->getHeight());
+        if (!bufferSizeExceedsLimit) {
+            buffer = std::make_shared<
+                    renderengine::ExternalTexture>(s.buffer, getRenderEngine(),
+                                                   renderengine::ExternalTexture::Usage::READABLE);
+        }
     }
+    ALOGE_IF(bufferSizeExceedsLimit,
+             "Attempted to create an ExternalTexture for layer %s that exceeds render target size "
+             "limit.",
+             layer->getDebugName());
     if (buffer) {
         const bool frameNumberChanged = what & layer_state_t::eFrameNumberChanged;
         const uint64_t frameNumber = frameNumberChanged
@@ -6186,6 +6193,13 @@
                                              const sp<IScreenCaptureListener>& captureListener) {
     ATRACE_CALL();
 
+    if (exceedsMaxRenderTargetSize(bufferSize.getWidth(), bufferSize.getHeight())) {
+        ALOGE("Attempted to capture screen with size (%" PRId32 ", %" PRId32
+              ") that exceeds render target size limit.",
+              bufferSize.getWidth(), bufferSize.getHeight());
+        return BAD_VALUE;
+    }
+
     // Loop over all visible layers to see whether there's any protected layer. A protected layer is
     // typically a layer with DRM contents, or have the GRALLOC_USAGE_PROTECTED set on the buffer.
     // A protected layer has no implication on whether it's secure, which is explicitly set by
diff --git a/services/surfaceflinger/SurfaceFlinger.h b/services/surfaceflinger/SurfaceFlinger.h
index c3d837e..c63225f 100644
--- a/services/surfaceflinger/SurfaceFlinger.h
+++ b/services/surfaceflinger/SurfaceFlinger.h
@@ -930,8 +930,9 @@
 
     void readPersistentProperties();
 
-    size_t getMaxTextureSize() const;
-    size_t getMaxViewportDims() const;
+    bool exceedsMaxRenderTargetSize(uint32_t width, uint32_t height) const {
+        return width > mMaxRenderTargetSize || height > mMaxRenderTargetSize;
+    }
 
     int getMaxAcquiredBufferCountForCurrentRefreshRate(uid_t uid) const;
 
@@ -1378,6 +1379,9 @@
 
     SurfaceFlingerBE mBE;
     std::unique_ptr<compositionengine::CompositionEngine> mCompositionEngine;
+    // mMaxRenderTargetSize is only set once in init() so it doesn't need to be protected by
+    // any mutex.
+    size_t mMaxRenderTargetSize{1};
 
     const std::string mHwcServiceName;
 
diff --git a/services/surfaceflinger/tests/ScreenCapture_test.cpp b/services/surfaceflinger/tests/ScreenCapture_test.cpp
index 6912fcf..ab2064e 100644
--- a/services/surfaceflinger/tests/ScreenCapture_test.cpp
+++ b/services/surfaceflinger/tests/ScreenCapture_test.cpp
@@ -515,7 +515,8 @@
 }
 
 TEST_F(ScreenCaptureTest, CaptureInvalidLayer) {
-    sp<SurfaceControl> redLayer = createLayer(String8("Red surface"), 60, 60, 0);
+    sp<SurfaceControl> redLayer = createLayer(String8("Red surface"), 60, 60,
+                                              ISurfaceComposerClient::eFXSurfaceBufferState);
 
     ASSERT_NO_FATAL_FAILURE(fillBufferQueueLayerColor(redLayer, Color::RED, 60, 60));
 
@@ -532,6 +533,21 @@
     ASSERT_EQ(NAME_NOT_FOUND, ScreenCapture::captureLayers(args, captureResults));
 }
 
+TEST_F(ScreenCaptureTest, CaptureTooLargeLayer) {
+    sp<SurfaceControl> redLayer = createLayer(String8("Red surface"), 60, 60);
+    ASSERT_NO_FATAL_FAILURE(fillBufferQueueLayerColor(redLayer, Color::RED, 60, 60));
+
+    Transaction().show(redLayer).setLayer(redLayer, INT32_MAX).apply(true);
+
+    LayerCaptureArgs captureArgs;
+    captureArgs.layerHandle = redLayer->getHandle();
+    captureArgs.frameScaleX = INT32_MAX / 60;
+    captureArgs.frameScaleY = INT32_MAX / 60;
+
+    ScreenCaptureResults captureResults;
+    ASSERT_EQ(BAD_VALUE, ScreenCapture::captureLayers(captureArgs, captureResults));
+}
+
 TEST_F(ScreenCaptureTest, CaptureSecureLayer) {
     sp<SurfaceControl> redLayer = createLayer(String8("Red surface"), 60, 60,
                                               ISurfaceComposerClient::eFXSurfaceBufferState);
diff --git a/services/surfaceflinger/tests/unittests/CompositionTest.cpp b/services/surfaceflinger/tests/unittests/CompositionTest.cpp
index 560f139..52a36a2 100644
--- a/services/surfaceflinger/tests/unittests/CompositionTest.cpp
+++ b/services/surfaceflinger/tests/unittests/CompositionTest.cpp
@@ -108,6 +108,8 @@
 
         mComposer = new Hwc2::mock::Composer();
         mFlinger.setupComposer(std::unique_ptr<Hwc2::Composer>(mComposer));
+
+        mFlinger.mutableMaxRenderTargetSize() = 16384;
     }
 
     ~CompositionTest() {
@@ -519,8 +521,6 @@
 
     static void setupLatchedBuffer(CompositionTest* test, sp<BufferQueueLayer> layer) {
         // TODO: Eliminate the complexity of actually creating a buffer
-        EXPECT_CALL(*test->mRenderEngine, getMaxTextureSize()).WillOnce(Return(16384));
-        EXPECT_CALL(*test->mRenderEngine, getMaxViewportDims()).WillOnce(Return(16384));
         status_t err =
                 layer->setDefaultBufferProperties(LayerProperties::WIDTH, LayerProperties::HEIGHT,
                                                   LayerProperties::FORMAT);
diff --git a/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h b/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h
index cf67593..3802e0d 100644
--- a/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h
+++ b/services/surfaceflinger/tests/unittests/TestableSurfaceFlinger.h
@@ -435,6 +435,7 @@
     auto& mutableTransactionFlags() { return mFlinger->mTransactionFlags; }
     auto& mutablePowerAdvisor() { return mFlinger->mPowerAdvisor; }
     auto& mutableDebugDisableHWC() { return mFlinger->mDebugDisableHWC; }
+    auto& mutableMaxRenderTargetSize() { return mFlinger->mMaxRenderTargetSize; }
 
     auto& mutableHwcDisplayData() { return getHwComposer().mDisplayData; }
     auto& mutableHwcPhysicalDisplayIdMap() { return getHwComposer().mPhysicalDisplayIdMap; }