sdk/fidl/fuchsia.audio.mixer/graph.fidl

// Copyright 2022 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

library fuchsia.audio.mixer;

using zx;
using fuchsia.mediastreams;
using fuchsia.audio.effects;
using fuchsia.audio;

alias NodeId = uint64;
alias ThreadId = uint64;
alias GainControlId = uint64;

/// By convention, the zero ID is never used.
const INVALID_ID uint64 = 0;

/// Maximum length of a `name` string.
const MAX_NAME_LENGTH uint32 = 256;

/// Maximum number of GainControls that can be attached to each graph edge.
const MAX_GAIN_STAGES_PER_EDGE uint32 = 32;

/// A factory for creating [`Graph`] protocols.
protocol GraphCreator {
    /// Creates a new graph that lives until the channel is closed.
    Create(resource table {
        /// Required.
        1: graph client_end:Graph;

        /// Name of this graph, used for diagnostics only.
        /// If specified, ideally this should be globally unique and have a
        /// printable CamelCase format, but uniqueness is not required.
        ///
        /// Optional. Empty if not specified.
        2: name string:MAX_NAME_LENGTH;

        /// Each graph has two threads dedicated to handling FIDL requests: a main
        /// thread, which handles time-insensitive requests, and a real-time FIDL
        /// thread, which handles time-sensitive requests. This field provides a
        /// deadline profile for the real-time FIDL thread.
        ///
        /// For example, all `[fuchsia.media2/StreamSink`] servers run on the
        /// real-time FIDL thread. Packets sent on this protocol must be read
        /// immediately, otherwise the caller may miss their deadline.
        ///
        /// Optional. If not specified, the real-time FIDL thread runs at a normal
        /// priority.
        3: realtime_fidl_thread_deadline_profile zx.handle:PROFILE;

        /// If specified, the graph will use synthetic clocks that are created
        /// and controlled over the given channel. This can be used to run the
        /// graph faster than real time, which is useful in integration tests.
        ///
        /// If specified, then ALL clocks used by this graph MUST be created by
        /// this realm OR by [`Graph.CreateGraphControlledReferenceClock`],
        /// which will use this realm internally.
        ///
        /// Optional. If not specified, the graph uses real clocks.
        4: synthetic_clock_realm server_end:SyntheticClockRealm;
    }) -> (table {}) error CreateGraphError;
};

/// A mixer Graph.
///
/// ## Directed Acyclic Graphs (DAGs)
///
/// Each graph includes a set of nodes connected into one or more DAGs. Audio
/// data flows from Producer nodes to Consumer nodes, and in between, may flow
/// through processing nodes, including Mixers, Splitters, and Custom nodes.
///
/// Audio is strongly typed: each node declares the audio encoding(s) it can
/// ingest as input(s) and the audio encoding(s) it can produce as output(s). It
/// is illegal to connect nodes with incompatible encodings.
///
/// ## Execution Model
///
/// Execution is driven by Consumers. Each Consumer wakes every N ms (N is
/// configurable per Consumer) and pulls N ms of audio from its inputs, which
/// pull audio from their inputs, and so on, up to the connected Producers. As
/// audio is pulled down the graph, it is processed and mixed into a single
/// stream that is written into the Consumer.
///
/// Each Consumer is attached to a Thread, which gives the Consumer a place to
/// do its work. Threads usually need deadline profiles to meet real-time
/// constraints. The client is responsible for creating Thread objects,
/// assigning Consumers to Threads, and attaching appropriate deadline profiles
/// when needed.
///
/// When the DAG includes nodes with multiple output edges, such as Splitters,
/// we can end up in a situation where two Consumers A and B share the same
/// dependencies (via the Splitter's input). If A and B run on different
/// threads, it's unclear which Thread should process those shared dependencies,
/// making it unclear how much deadline capacity is required by each thread. To
/// avoid this problem, we partition the DAG so that each Consumer is the root
/// of an inverted tree. At nodes with multiple output edges, such as Splitters,
/// we partition the node into a hidden Consumer node (which drives the inputs)
/// and hidden Producer nodes (which drive the outputs). When the client creates
/// a Splitter, they must assign a Thread to the Splitter's hidden Consumer.
/// This ensures that each node is processed on a unique thread, making it
/// simpler to analyze the needed capacity for each Thread.
///
/// ## Method Semantics
///
/// Methods will be executed sequentially in the order they are called. Method
/// calls can be pipelined, but if more than an implementation-defined number of
/// requests are in flight at one time, the server reserves the right to assume
/// a DoS attack and close the connection.
///
/// Most methods use a minimal set of arguments plus an "options" table to allow
/// for extensibility.
///
/// ## IDs and names
///
/// Every object is identified by a numeric `id`. Within each object type
/// (Nodes, Threads, and GainControls) IDs are guaranteed to be unique. Old IDs
/// for deleted objects will never be reused for new objects.
///
/// Every object has an optional string `name`. If specified, ideally this name
/// should be unique within the [`Graph`] and have a printable CamelCase format,
/// but uniqueness is not required. Names are used for developer-visible
/// diagnostics only -- they do not need to be unique. Duplicate names can at
/// worst lead to potentially-confusing diagnostics. IDs, not names, should be
/// used when unique identification is required.
///
/// ## Clocks
///
/// Each audio stream is associated with a *reference clock*. Different streams
/// can use different clocks. Any two clocks can differ in both value (the
/// current time) and the rate, where the rate may [change over
/// time](https://fuchsia.dev/fuchsia-src/reference/syscalls/clock_update) as
/// long as the clock remains [continuous and
/// monotonic](https://fuchsia.dev/fuchsia-src/reference/kernel_objects/clock).
/// This reflects many real situations. For example, a speaker may have an
/// internal clock separate from the CPU's physical clock. Or, a stream may
/// originate from some other computer on the network whose clock is not
/// precisely synchronized to our local clock.
///
/// Every node must specify the reference clock used by the node's output
/// streams, except for Consumers, which must specify a clock for the Consumer's
/// input stream. To connect two streams that use different clocks, we must
/// translate one stream onto the other stream's clock. This is done at Mixer
/// nodes, which use sample rate conversion (SRC) to translate input streams
/// onto the Mixer's output reference clock.
///
/// Reference clocks can change rate over time. These rate changes are typically
/// controlled by the client. If the client doesn't need precise control over
/// reference clocks, a cheaper option is to use Graph-controlled clocks (see
/// [`Graph.CreateGraphControlledReferenceClock`]), which can avoid a
/// potentially-expensive SRC in many cases. For example, if a Producer flows to
/// a Consumer, where the Producer uses a Graph-controlled clock and the
/// Consumer uses a client-controlled clock, the `Graph` will adjust the
/// Producer clock's rate to synchronize the Producer and Consumer clocks,
/// eliminating the need for SRC.
protocol Graph {
    /// Creates a Producer node with the given options.
    ///
    /// Producer nodes generate audio which can be consumed by other nodes. For
    /// example, a Producer node might encapsulate audio coming from an
    /// application or from a microphone. Producer nodes cannot have any input
    /// edges but may have one or more output edges.
    CreateProducer(resource table {
        /// Name of this node, used for diagnostics only. See "IDs and names" in the
        /// comments for [`Graph`].
        ///
        /// Optional. Empty if not specified.
        1: name string:MAX_NAME_LENGTH;

        /// Direction of audio data produced by this node.
        ///
        /// Required.
        2: direction PipelineDirection;

        /// Data source for this producer.
        ///
        /// Required.
        3: data_source ProducerDataSource;
    }) -> (table {
        /// ID of the newly-created node. Guaranteed to be unique.
        1: id NodeId;
    }) error CreateNodeError;

    /// Creates a Consumer node with the given options.
    ///
    /// Consumer nodes write audio to a sink. For example, a Consumer node might
    /// encapsulate audio being written to a speaker or to an application (which
    /// may be capturing audio from a microphone). Consumer nodes can have at
    /// most one input edge.
    ///
    /// Audio pipelines are driven by Consumers. Each Consumer is attached to a
    /// Thread, which gives the Consumer a place to do work. A Consumer wakes
    /// every `N` ms (`N` is configurable), pulls `N` ms of audio from its input
    /// edge, then writes that audio to the Consumer's sink.
    ///
    /// For more details, see "Execution Model" under the description for
    /// [`Graph`].
    CreateConsumer(resource table {
        /// Name of this node, used for diagnostics only. See "IDs and names" in the
        /// comments for [`Graph`].
        ///
        /// Optional. Empty if not specified.
        1: name string:MAX_NAME_LENGTH;

        /// Direction of audio data consumed by this node.
        ///
        /// Required.
        2: direction PipelineDirection;

        /// Data source for this consumer.
        ///
        /// Required.
        3: data_source ConsumerDataSource;

        /// Configuration for the consumer that is common across types.
        ///
        /// Required.
        4: options ConsumerOptions;
    }) -> (table {
        /// ID of the newly-created node. Guaranteed to be unique.
        1: id NodeId;
    }) error CreateNodeError;

    /// Creates a Mixer node with the given options.
    ///
    /// Mixer nodes combine multiple PCM input streams into a single PCM output
    /// streams. Mixers apply format conversion and sample rate conversion to
    /// the input streams to produce an output stream with a fixed format.
    CreateMixer(resource table {
        /// Name of this node, used for diagnostics only. See "IDs and names" in the
        /// comments for [`Graph`].
        ///
        /// Optional. Empty if not specified.
        1: name string:MAX_NAME_LENGTH;

        /// Direction of audio data produced by this node.
        ///
        /// Required.
        2: direction PipelineDirection;

        /// Encoding of the Mixer's output stream.
        ///
        /// Required.
        3: output_format fuchsia.mediastreams.AudioFormat;

        /// Clock for this node's output stream. The clock must be
        /// ZX_CLOCK_OPT_MONOTONIC and ZX_CLOCK_OPT_CONTINUOUS with ZX_RIGHT_READ.
        /// See "Clocks" under the description for [`Graph`].
        ///
        /// Required.
        4: output_reference_clock zx.handle:CLOCK;

        /// Size of the internal mix buffer. This defines the maximum number of
        /// output frames that can be mixed at one time.
        ///
        /// Optional. If not specified, a default size is used.
        5: output_buffer_frame_count uint64;
    }) -> (table {
        /// ID of the newly-created node. Guaranteed to be unique.
        1: id NodeId;
    }) error CreateNodeError;

    /// Creates a Splitter node with the given options.
    ///
    /// Splitter nodes split a single input stream into multiple output streams
    /// that are clones of the input stream. For more details, see "Execution
    /// Model" under the description for [`Graph`].
    CreateSplitter(resource table {
        /// Name of this node, used for diagnostics only. See "IDs and names" in the
        /// comments for [`Graph`].
        ///
        /// Optional. Empty if not specified.
        1: name string:MAX_NAME_LENGTH;

        /// Direction of audio data consumed by this node.
        ///
        /// Required.
        2: direction PipelineDirection;

        /// Encoding of the Splitter's input and output streams.
        ///
        /// Required.
        3: format fuchsia.mediastreams.AudioFormat;

        /// Splitters are composite nodes that process their input through a hidden
        /// Consumer node. For more details on this behavior, see "Execution Model"
        /// under the description for [`Graph`].
        ///
        /// Required.
        4: consumer ConsumerOptions;

        /// Clock for this node's input and output streams. The clock must be
        /// ZX_CLOCK_OPT_MONOTONIC and ZX_CLOCK_OPT_CONTINUOUS with ZX_RIGHT_READ.
        /// See "Clocks" under the description for [`Graph`].
        ///
        /// Required.
        5: reference_clock zx.handle:CLOCK;
    }) -> (table {
        /// ID of the newly-created node. Guaranteed to be unique.
        1: id NodeId;
    }) error CreateNodeError;

    /// Creates a Custom node with the given options.
    ///
    /// Custom nodes apply custom effects to one or more input streams,
    /// producing one or more output streams. The effects are implemented
    /// out-of-process via a call to a FIDL interface.
    ///
    /// Custom nodes are composite nodes that encapsulate a fixed number of
    /// inputs and outputs. We assign an ID to each of these input and output
    /// slots -- see [`CustomNodeProperties`]. This allows creating edges that
    /// target a specific slot. For example, a node that implements AEC will
    /// have loopback and microphone input slots and the caller will need to
    /// connect each input slot to an appropriate source. The caller can do this
    /// by calling `CreateEdge` using a specific [`NodeId`] from
    /// [`CustomNodeProperties.input_ids`]. These internal IDs cannot be
    /// deleted, except by deleting the entire Custom node.
    ///
    /// The returned `id` describes the composite node. Passing `id` to
    /// [`DeleteNode`] will delete the composite node as well as any internal
    /// input and output nodes. The returned `id` cannot be used in
    /// [`CreateEdge`]. Edges must target a specific input or output slot as
    /// described above.
    CreateCustom(resource table {
        /// Name of this node, used for diagnostics only. See "IDs and names" in the
        /// comments for [`Graph`].
        ///
        /// Optional. Empty if not specified.
        1: name string:MAX_NAME_LENGTH;

        /// Direction of audio data consumed and produced by this node.
        ///
        /// Required.
        2: direction PipelineDirection;

        /// Description of the out-of-process effects processor.
        /// This includes a description of the node's input and output edges.
        ///
        /// Required.
        3: config fuchsia.audio.effects.ProcessorConfiguration;

        /// Configuration of the hidden Consumer node.
        ///
        /// If the Custom node has multiple outputs, this field is required. The
        /// hidden Consumer reads the Custom node's input(s) and writes its outputs.
        /// It is used to partition the graph as described under "Execution Model"
        /// under the description for [`Graph`].
        ///
        /// If the Custom node has a single output, then a hidden Consumer is not
        /// used and this field must not be specified.
        4: consumer ConsumerOptions;

        /// Clock for this node's input and output streams. The clock must be
        /// ZX_CLOCK_OPT_MONOTONIC and ZX_CLOCK_OPT_CONTINUOUS with ZX_RIGHT_READ.
        /// See "Clocks" under the description for [`Graph`].
        ///
        /// Required.
        5: reference_clock zx.handle:CLOCK;
    }) -> (resource table {
        /// ID of the newly-created node. Guaranteed to be unique.
        1: id NodeId;

        /// Additional properties of the newly-created node.
        2: node_properties CustomNodeProperties;
    }) error CreateNodeError;

    /// Deletes the given node.
    ///
    /// The node must exist. `DeleteNode(x)` will delete all incoming and
    /// outgoing edges of node `x` before deleting the node itself. Once the node
    /// has been deleted it cannot be mentioned by any future method calls.
    DeleteNode(table {
        /// ID of the node to delete.
        1: id NodeId;
    }) -> (table {}) error DeleteNodeError;

    /// Creates an edge from the source node to the destination node.
    ///
    /// Both nodes must exist. The source's output encoding must be supported by
    /// the destination node. With the exception of Mixer nodes, most nodes
    /// require inputs to have a specific (fixed) encoding.
    ///
    /// The source's output streams must use the same reference clock as the
    /// destinations input stream, unless the destination is a Mixer, in which
    /// case the source's output stream can use any clock.
    ///
    /// If the source's direction is `INPUT`, the dest's direction cannot be
    /// `OUTPUT`. See [`PipelineDirection`] for additional discussion.
    CreateEdge(resource table {
        /// ID of the destination node.
        /// Required.
        1: dest_id NodeId;

        /// ID of the source node.
        /// Required.
        2: source_id NodeId;

        /// This selects the sampler to use when performing sample rate conversion
        /// on the input. Valid only when the dest node is a Mixer.
        ///
        /// Optional. If not specified, a default sampler is selected.
        3: mixer_sampler Sampler;

        /// Gains to apply to this edge. Since gain is applied by Mixer nodes,
        /// either the source or dest node must be a Mixer.
        ///
        /// Optional. If empty, no gain is applied.
        4: gain_stages vector<GainControlId>:MAX_GAIN_STAGES_PER_EDGE;
    }) -> (table {}) error CreateEdgeError;

    /// Deletes the edge connecting the source node to the destination node.
    ///
    /// The edge must exist.
    DeleteEdge(table {
        /// ID of the destination node.
        1: dest_id NodeId;

        /// ID of the source node.
        2: source_id NodeId;
    }) -> (table {}) error DeleteEdgeError;

    /// Creates a thread.
    ///
    /// Each `CreateThread` call creates a new thread in the mixer service. This
    /// new thread will be used to process audio for all Consumer nodes assigned
    /// to this thread.
    ///
    /// For more details, see "Execution Model" under the description for
    /// [`Graph`].
    CreateThread(resource table {
        /// Name of this thread, used for diagnostics only. See "IDs and names" in the
        /// comments for [`Graph`].
        ///
        /// Optional. Empty if not specified.
        1: name string:MAX_NAME_LENGTH;

        /// Deadline profile to apply to this thread.
        ///
        /// Optional. If not specified, the thread runs at a normal priority.
        2: deadline_profile zx.handle:PROFILE;
    }) -> (table {
        /// ID of the newly-created thread. Guaranteed to be unique.
        1: id ThreadId;
    }) error CreateThreadError;

    /// Deletes the given thread.
    ///
    /// The thread must exist. A thread cannot be deleted until all Consumer
    /// nodes assigned to that thread have been deleted.
    DeleteThread(table {
        /// ID of the thread to delete.
        /// Required.
        1: id ThreadId;
    }) -> (table {}) error DeleteThreadError;

    /// Creates a GainControl.
    ///
    /// A GainControl controls gain that should be applied to an audio stream.
    /// GainControls can be attached to edges going into and out of a Mixer node.
    /// Gain can be set to a specific value (e.g. in decibels) or it can be
    /// muted.
    ///
    /// By default, a GainControl applies no gain and is not muted.
    CreateGainControl(resource table {
        /// Name of this GainControl, used for diagnostics only. See "IDs and names"
        /// in the comments for [`Graph`].
        ///
        /// Optional. Empty if not specified.
        1: name string:MAX_NAME_LENGTH;

        /// Interface which controls this GainControl.
        ///
        /// Required.
        2: control server_end:fuchsia.audio.GainControl;
    }) -> (table {
        /// ID of the newly-created GainControl. Guaranteed to be unique.
        1: id GainControlId;
    }) error CreateGainControlError;

    /// Deletes the given GainControl.
    ///
    /// The GainControl must exist. A GainControl cannot be deleted until all
    /// associated edges have been deleted.
    DeleteGainControl(table {
        /// ID of the GainControl to delete.
        /// Required.
        1: id GainControlId;
    }) -> (table {}) error DeleteGainControlError;

    /// Creates a graph-controlled reference clock.
    ///
    /// The returned clock has `ZX_RIGHT_READ` and `ZX_RIGHT_DUPLICATE` but not
    /// `ZX_RIGHT_WRITE`. The clock may be duplicated and used wherever a
    /// reference clock is needed. The graph will rate change this clock until
    /// the clock (or a duplicate of the clock) is passed to
    /// [`ForgetGraphControlledReferenceClock`].
    ///
    /// For more details, see "Clocks" under the description for [`Graph`].
    ///
    /// * error Error from `zx_clock_create`
    CreateGraphControlledReferenceClock() -> (resource table {
        /// The new clock.
        1: reference_clock zx.handle:CLOCK;
    }) error zx.status;

    /// Forgets about a graph-controlled reference clock.
    ///
    /// The clock must have been previously created by
    /// [`CreateGraphControlledReferenceClock`]. The clock will not be deleted (other
    /// duplicates will still exist) however it will no longer be adjusted by
    /// this Graph (or by anyone else, since this Graph was the only entity with
    /// `ZX_RIGHT_WRITE`).
    ///
    /// * error `ZX_ERR_NOT_FOUND` if `reference_clock` was not created by
    /// [`CreateGraphControlledReferenceClock`] or if was already forgotten
    ForgetGraphControlledReferenceClock(resource table {
        /// The clock to forget.
        /// Required.
        1: reference_clock zx.handle:CLOCK;
    }) -> (table {}) error zx.status;

    // TODO(fxbug.dev/87651): Start/Stop methods to control Producer and Consumer nodes
    // TODO(fxbug.dev/87651): Watch method to observe delay (aka lead time) of a node
};

/// Type of errors return by [`CreateGraph`].
type CreateGraphError = flexible enum {};

/// Type of errors return by CreateNode methods.
type CreateNodeError = flexible enum {
    /// A provided `zx.handle:CLOCK` was invalid or had incorrect rights.
    INVALID_CLOCK = 1;
};

/// Type of errors return by [`DeleteNode`].
type DeleteNodeError = flexible enum {
    /// The given `id` is invalid.
    DOES_NOT_EXIST = 1;
};

/// Type of errors return by [`CreateEdge`].
type CreateEdgeError = flexible enum {
    /// The given `dest_id` is invalid.
    INVALID_DEST_ID = 1;

    /// The dest does not support an additional input.
    DEST_HAS_TOO_MANY_INPUTS = 2;

    /// The given `source_id` is invalid.
    INVALID_SOURCE_ID = 3;

    /// The source does not support an additional output.
    SOURCE_HAS_TOO_MANY_OUTPUTS = 4;

    /// The source's output is not compatible with the dest.
    INCOMPATIBLE_FORMATS = 5;

    /// The source and dest are already connected.
    ALREADY_CONNECTED = 6;

    /// This edge would create a cycle.
    CYCLE = 7;
};

/// Type of errors return by [`DeleteEdge`].
type DeleteEdgeError = flexible enum {
    /// The given `dest_id` is invalid.
    INVALID_DEST_ID = 1;

    /// The given `source_id` is invalid.
    INVALID_SOURCE_ID = 2;

    /// The edge does not exist.
    EDGE_NOT_FOUND = 3;
};

/// Type of errors return by [`CreateThread`].
type CreateThreadError = flexible enum {};

/// Type of errors return by [`DeleteThread`].
type DeleteThreadError = flexible enum {
    /// The given `id` is invalid.
    INVALID_ID = 1;

    /// There are still Consumer nodes assigned to this thread. These Consumers
    /// must be deleted before the Thread can be deleted.
    STILL_IN_USE = 2;
};

/// Type of errors return by [`CreateGainControl`].
type CreateGainControlError = flexible enum {};

/// Type of errors return by [`DeleteGainControl`].
type DeleteGainControlError = flexible enum {
    /// The given `id` is invalid.
    INVALID_ID = 1;

    /// There are still edges using this GainControl. These edges must be deleted
    /// before the GainControl can be deleted.
    STILL_IN_USE = 2;
};