sync: Simplify SignaledSemaphores::Unsignal v2

Unsignal is only called on the child object (not on the
parent signaled_semaphores_) => prev_ != null. Added assert.
diff --git a/layers/sync/sync_submit.cpp b/layers/sync/sync_submit.cpp
index 94d2318..42ca125 100644
--- a/layers/sync/sync_submit.cpp
+++ b/layers/sync/sync_submit.cpp
@@ -24,15 +24,6 @@
 
 bool AcquiredImage::Invalid() const { return vvl::StateObject::Invalid(image); }
 
-// This is a const method, force the returned value to be const
-std::shared_ptr<const SignaledSemaphores::Signal> SignaledSemaphores::GetPrev(VkSemaphore sem) const {
-    std::shared_ptr<Signal> prev_state;
-    if (prev_) {
-        prev_state = GetMapped(prev_->signaled_, sem);
-    }
-    return prev_state;
-}
-
 SignaledSemaphores::Signal::Signal(const std::shared_ptr<const vvl::Semaphore>& sem_state_,
                                    const std::shared_ptr<QueueBatchContext>& batch_, const SyncExecScope& exec_scope_)
     : sem_state(sem_state_), batch(batch_), first_scope({batch->GetQueueId(), exec_scope_}) {
@@ -91,25 +82,19 @@
 }
 
 std::shared_ptr<const SignaledSemaphores::Signal> SignaledSemaphores::Unsignal(VkSemaphore sem) {
+    assert(prev_ != nullptr);
     std::shared_ptr<const Signal> unsignaled;
     const auto found_it = signaled_.find(sem);
     if (found_it != signaled_.end()) {
         // Move the unsignaled singal out from the signaled list, but keep the shared_ptr as the caller needs the contents for
         // a bit.
         unsignaled = std::move(found_it->second);
-        if (!prev_) {
-            // No parent, not need to keep the entry
-            // IFF (prev_)  leave the entry in the leaf table as we use it to export unsignal to prev_ during record phase
-            signaled_.erase(found_it);
-        }
-    } else if (prev_) {
+    } else {
         // We can't unsignal prev_ because it's const * by design.
         // We put in an empty placeholder
         signaled_.emplace(sem, std::shared_ptr<Signal>());
-        unsignaled = GetPrev(sem);
+        unsignaled = GetMapped(prev_->signaled_, sem);
     }
-    // NOTE: No else clause. Because if we didn't find it, and there's no previous, this indicates an error,
-    // but CoreChecks should have reported it
 
     // If unsignaled is null, there was a missing pending semaphore, and that's also issue CoreChecks reports
     return unsignaled;
diff --git a/layers/sync/sync_submit.h b/layers/sync/sync_submit.h
index 7f8412a..f54fda2 100644
--- a/layers/sync/sync_submit.h
+++ b/layers/sync/sync_submit.h
@@ -89,7 +89,6 @@
   private:
     void Import(VkSemaphore sem, std::shared_ptr<Signal> &&move_from);
     void Reset();
-    std::shared_ptr<const Signal> GetPrev(VkSemaphore sem) const;
 
   private:
     vvl::unordered_map<VkSemaphore, std::shared_ptr<Signal>> signaled_;