kernel/vm/page_source.cpp

// Copyright 2018 The Fuchsia Authors
//
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file or at
// https://opensource.org/licenses/MIT

#include <fbl/auto_lock.h>
#include <kernel/lockdep.h>
#include <ktl/move.h>
#include <trace.h>
#include <vm/page_source.h>

#define LOCAL_TRACE 0

PageSource::PageSource() {
    LTRACEF("%p\n", this);
}

PageSource::~PageSource() {
    LTRACEF("%p\n", this);
    DEBUG_ASSERT(detached_);
    DEBUG_ASSERT(closed_);
}

void PageSource::Detach() {
    canary_.Assert();
    LTRACEF("%p\n", this);
    Guard<fbl::Mutex> guard{&page_source_mtx_};
    if (detached_) {
        return;
    }

    detached_ = true;

    // Cancel read requests (which everything for now)
    while (!outstanding_requests_.is_empty()) {
        auto req = outstanding_requests_.pop_front();
        LTRACEF("dropping request with offset %lx\n", req->offset_);

        // Tell the clients the request is complete - they'll fail when they try
        // asking for the requested pages again after failing to find the pages
        // for this request.
        CompleteRequestLocked(req);
    }
    OnDetach();
}


void PageSource::Close() {
    canary_.Assert();
    LTRACEF("%p\n", this);
    // TODO: Close will have more meaning once writeback is implemented
    Detach();

    Guard<fbl::Mutex> guard{&page_source_mtx_};
    if (!closed_) {
        closed_ = true;
        OnClose();
    }
}

void PageSource::OnPagesSupplied(uint64_t offset, uint64_t len) {
    canary_.Assert();
    LTRACEF("%p offset %lx, len %lx\n", this, offset, len);
    uint64_t end = offset + len;

    Guard<fbl::Mutex> guard{&page_source_mtx_};
    if (detached_) {
        return;
    }

    // The first possible request we could fulfill is the one with the smallest
    // address that is at least offset. Then keep looking as long as the target's
    // request's start offset is less than the supply end.
    auto start = outstanding_requests_.lower_bound(offset);
    while (start.IsValid() && start->offset_ < end) {
        auto cur = start;
        ++start;

        LTRACEF("%p, signaling %lx\n", this, cur->offset_);

        // Notify anything waiting on this page
        CompleteRequestLocked(outstanding_requests_.erase(cur));
    }
}

zx_status_t PageSource::GetPage(uint64_t offset, PageRequest* request,
                                vm_page_t** const page_out, paddr_t* const pa_out) {
    canary_.Assert();
    ASSERT(request);

    Guard<fbl::Mutex> guard{&page_source_mtx_};
    LTRACEF("%p offset %lx, %lu\n", this, offset, get_current_thread()->user_tid);
    if (detached_) {
        return ZX_ERR_NOT_FOUND;
    }

    if (GetPage(offset, page_out, pa_out)) {
        return ZX_OK;
    }

    request->Init(fbl::RefPtr<PageSource>(this), offset);

    LTRACEF("%p %p\n", this, request);
    auto overlap = outstanding_requests_.find(request->offset_);
    if (overlap.IsValid()) {
        overlap->overlap_.push_back(request);
    } else {
        list_clear_node(&request->read_request_.node);
        request->read_request_.offset = request->offset_;
        request->read_request_.length = PAGE_SIZE;

        GetPageAsync(&request->read_request_);
        outstanding_requests_.insert(request);
    }

    return ZX_ERR_SHOULD_WAIT;
}

void PageSource::CompleteRequestLocked(PageRequest* req) {
    // Take the request back from the callback before waking
    // up the corresponding thread.
    ClearAsyncRequest(&req->read_request_);

    while (!req->overlap_.is_empty()) {
        auto waiter = req->overlap_.pop_front();
        waiter->offset_ = UINT64_MAX;
        event_signal(&waiter->event_, true);
    }
    req->offset_ = UINT64_MAX;
    event_signal(&req->event_, true);
}

void PageSource::CancelRequest(PageRequest* request) {
    canary_.Assert();
    Guard<fbl::Mutex> guard{&page_source_mtx_};
    LTRACEF("%p %lx\n", this, request->offset_);

    if (request->offset_ == UINT64_MAX) {
        return;
    }

    if (static_cast<fbl::DoublyLinkedListable<PageRequest*>*>(request)->InContainer()) {
        LTRACEF("Overlap node\n");
        // This node is overlapping some other node, so just remove the request
        auto main_node = outstanding_requests_.find(request->offset_);
        main_node->overlap_.erase(*request);
    } else if (!request->overlap_.is_empty()) {
        LTRACEF("Outstanding with overlap\n");
        // This node is an outstanding request with overlap, so replace it with the
        // first overlap node.
        auto new_node = request->overlap_.pop_front();

        new_node->overlap_.swap(request->overlap_);
        new_node->offset_ = request->offset_;

        list_clear_node(&new_node->read_request_.node);
        new_node->read_request_.offset = request->offset_;
        new_node->read_request_.length = PAGE_SIZE;

        outstanding_requests_.erase(*request);
        outstanding_requests_.insert(new_node);

        SwapRequest(&request->read_request_, &new_node->read_request_);
    } else if (static_cast<fbl::WAVLTreeContainable<PageRequest*>*>(request)->InContainer()) {
        LTRACEF("Outstanding no overlap\n");
        // This node is an outstanding request with no overlap
        outstanding_requests_.erase(*request);
        ClearAsyncRequest(&request->read_request_);
    }

    request->offset_ = UINT64_MAX;
}

PageRequest::~PageRequest() {
    if (offset_ != UINT64_MAX) {
        src_->CancelRequest(this);
    }
}

void PageRequest::Init(fbl::RefPtr<PageSource> src, uint64_t offset) {
    DEBUG_ASSERT(offset_ == UINT64_MAX);
    offset_ = offset;
    src_ = ktl::move(src);
    event_ = EVENT_INITIAL_VALUE(event_, 0, EVENT_FLAG_AUTOUNSIGNAL);
}

zx_status_t PageRequest::Wait() {
    zx_status_t status = src_->WaitOnEvent(&event_);
    if (status != ZX_OK) {
        src_->CancelRequest(this);
    }
    return status;
}