| // Copyright 2022 The Fuchsia Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| library fuchsia.audio.mixer; |
| |
| using zx; |
| |
| alias NodeId = uint64; |
| alias ThreadId = uint64; |
| alias GainStageId = uint64; |
| |
| /// By convention, the zero ID is never used. |
| const INVALID_ID uint64 = 0; |
| |
| /// Maximum length of a `name` string. |
| const MAX_NAME_LENGTH uint32 = 256; |
| |
| /// A factory for creating [`Graph`] protocols. |
| protocol GraphCreator { |
| /// Creates a new graph that lives until the channel is closed. |
| Create(resource struct { |
| graph client_end:Graph; |
| options CreateGraphOptions; |
| }) -> (struct {}) error CreateGraphError; |
| }; |
| |
| /// A mixer Graph. |
| /// |
| /// ## Directed Acyclic Graphs (DAGs) |
| /// |
| /// Each graph includes a set of nodes connected into one or more DAGs. Audio |
| /// data flows from Producer nodes to Consumer nodes, and in between, may flow |
| /// through processing nodes, including Mixers, Splitters, and Custom nodes. |
| /// |
| /// Audio is strongly typed: each node declares the audio encoding(s) it can |
| /// ingest as input(s) and the audio encoding(s) it can produce as output(s). It |
| /// is illegal to connect nodes with incompatible encodings. |
| /// |
| /// ## Execution Model |
| /// |
| /// Execution is driven by Consumers. Each Consumer wakes every N ms (N is |
| /// configurable per Consumer) and pulls N ms of audio from its inputs, which |
| /// pull audio from their inputs, and so on, up to the connected Producers. As |
| /// audio is pulled down the graph, it is processed and mixed into a single |
| /// stream that is written into the Consumer. |
| /// |
| /// Each Consumer is attached to a Thread, which gives the Consumer a place to |
| /// do its work. Threads usually need deadline profiles to meet real-time |
| /// constraints. The client is responsible for creating Thread objects, |
| /// assigning Consumers to Threads, and attaching appropriate deadline profiles |
| /// when needed. |
| /// |
| /// When the DAG includes nodes with multiple output edges, such as Splitters, |
| /// we can end up in a situation where two Consumers A and B share the same |
| /// dependencies (via the Splitter's input). If A and B run on different |
| /// threads, it's unclear which Thread should process those shared dependencies, |
| /// making it unclear how much deadline capacity is required by each thread. To |
| /// avoid this problem, we partition the DAG so that each Consumer is the root |
| /// of an inverted tree. At nodes with multiple output edges, such as Splitters, |
| /// we partition the node into a hidden Consumer node (which drives the inputs) |
| /// and hidden Producer nodes (which drive the outputs). When the client creates |
| /// a Splitter, they must assign a Thread to the Splitter's hidden Consumer. |
| /// This ensures that each node is processed on a unique thread, making it |
| /// simpler to analyze the needed capacity for each Thread. |
| /// |
| /// ## Method Semantics |
| /// |
| /// Methods will be executed sequentially in the order they are called. Method |
| /// calls can be pipelined, but if more than an implementation-defined number of |
| /// requests are in flight at one time, the server reserves the right to assume |
| /// a DoS attack and close the connection. |
| /// |
| /// Most methods use a minimal set of arguments plus an "options" table to allow |
| /// for extensibility. |
| /// |
| /// ## IDs and names |
| /// |
| /// Every object is identified by a numeric `id`. Within each object type |
| /// (Nodes, Threads, and GainStages) IDs are guaranteed to be unique. Old IDs |
| /// for deleted objects will never be reused for new objects. |
| /// |
| /// Every object has an optional string `name`. If specified, ideally this name |
| /// should be unique within the [`Graph`] and have a printable CamelCase format, |
| /// but this is not required. Names are used for developer-visible diagnostics |
| /// only -- they do not need to be unique. Duplicate names can at worst lead to |
| /// potentially-confusing diagnostics. IDs, not names, should be used when |
| /// unique identification is required. |
| /// |
| /// ## Clocks |
| /// |
| /// Each audio stream is associated with a *reference clock*. Different streams |
| /// can use different clocks. Any two clocks can differ in both value (the |
| /// current time) and the rate, where the rate may [change over |
| /// time](https://fuchsia.dev/fuchsia-src/reference/syscalls/clock_update) as |
| /// long as the clock remains [continuous and |
| /// monotonic](https://fuchsia.dev/fuchsia-src/reference/kernel_objects/clock). |
| /// This reflects many real situations. For example, a speaker may have an |
| /// internal clock separate from the CPU's physical clock. Or, a stream may |
| /// originate from some other computer on the network whose clock is not |
| /// precisely synchronized to our local clock. |
| /// |
| /// Every node must specify the reference clock used by the node's output |
| /// streams, except for Consumers, which must specify a clock for the Consumer's |
| /// input stream. To connect two streams that use different clocks, we must |
| /// translate one stream onto the other stream's clock. This is done at Mixer |
| /// nodes, which use sample rate conversion (SRC) to translate input streams |
| /// onto the Mixer's output reference clock. |
| /// |
| /// Reference clocks can change rate over time. These rate changes are typically |
| /// controlled by the client. If the client doesn't need precise control over |
| /// reference clocks, a cheaper option is to use Graph-controlled clocks (see |
| /// [`Graph.CreateGraphControlledReferenceClock`]), which can avoid a |
| /// potentially-expensive SRC in many cases. For example, if a Producer flows to |
| /// a Consumer, where the Producer uses a Graph-controlled clock and the |
| /// Consumer uses a client-controlled clock, the `Graph` will adjust the |
| /// Producer clock's rate to synchronize the Producer and Consumer clocks, |
| /// eliminating the need for SRC. |
| protocol Graph { |
| /// Creates a Producer node with the given options. |
| /// |
| /// Producer nodes generate audio which can be consumed by other nodes. For |
| /// example, a Producer node might encapsulate audio coming from an |
| /// application or from a microphone. Producer nodes cannot have any input |
| /// edges but may have one or more output edges. |
| /// |
| /// + `options` Extensible options for creating the node. |
| /// - `id` ID of the newly-created node. Guaranteed to be unique. |
| /// * error Reason the node could not be created. |
| CreateProducer(resource struct { |
| options CreateProducerOptions; |
| }) -> (struct { |
| id NodeId; |
| }) error CreateNodeError; |
| |
| /// Creates a Consumer node with the given options. |
| /// |
| /// Consumer nodes write audio to a sink. For example, a Consumer node might |
| /// encapsulate audio being written to a speaker or to an application (which |
| /// may be capturing audio from a microphone). Consumer nodes can have at |
| /// most one input edge. |
| /// |
| /// Audio pipelines are driven by Consumers. Each Consumer is attached to a |
| /// Thread, which gives the Consumer a place to do work. A Consumer wakes |
| /// every `N` ms (`N` is configurable), pulls `N` ms of audio from its input |
| /// edge, then writes that audio to the Consumer's sink. |
| /// |
| /// For more details, see "Execution Model" under the description for |
| /// [`Graph`]. |
| /// |
| /// + `options` Extensible options for creating the node. |
| /// - `id` ID of the newly-created node. Guaranteed to be unique. |
| /// * error Reason the node could not be created. |
| CreateConsumer(resource struct { |
| options CreateConsumerOptions; |
| }) -> (struct { |
| id NodeId; |
| }) error CreateNodeError; |
| |
| /// Creates a Mixer node with the given options. |
| /// |
| /// Mixer nodes combine multiple PCM input streams into a single PCM output |
| /// streams. Mixers apply format conversion and sample rate conversion to |
| /// the input streams to produce an output stream with a fixed format. |
| /// |
| /// + `options` Extensible options for creating the node. |
| /// - `id` ID of the newly-created node. Guaranteed to be unique. |
| /// * error Reason the node could not be created. |
| CreateMixer(resource struct { |
| options CreateMixerOptions; |
| }) -> (struct { |
| id NodeId; |
| }) error CreateNodeError; |
| |
| /// Creates a Splitter node with the given options. |
| /// |
| /// Splitter nodes split a single input stream into multiple output streams |
| /// that are clones of the input stream. For more details, see "Execution |
| /// Model" under the description for [`Graph`]. |
| /// |
| /// + `options` Extensible options for creating the node. |
| /// - `id` ID of the newly-created node. Guaranteed to be unique. |
| /// * error Reason the node could not be created. |
| CreateSplitter(resource struct { |
| options CreateSplitterOptions; |
| }) -> (struct { |
| id NodeId; |
| }) error CreateNodeError; |
| |
| /// Creates a Custom node with the given options. |
| /// |
| /// Custom nodes apply custom effects to one or more input streams, |
| /// producing one or more output streams. The effects are implemented |
| /// out-of-process via a call to a FIDL interface. |
| /// |
| /// Custom nodes are composite nodes that encapsulate a fixed number of |
| /// inputs and outputs. We assign an ID to each of these input and output |
| /// slots -- see [`CustomNodeProperties`]. This allows creating edges that |
| /// target a specific slot. For example, a node that implements AEC will |
| /// have loopback and microphone input slots and the caller will need to |
| /// connect each input slot to an appropriate source. The caller can do this |
| /// by calling `CreateEdge` using a specific [`NodeId`] from |
| /// [`CustomNodeProperties.input_ids`]. These internal IDs cannot be |
| /// deleted, except by deleting the entire Custom node. |
| /// |
| /// The returned `id` describes the composite node. Passing `id` to |
| /// [`DeleteNode`] will delete the composite node as well as any internal |
| /// input and output nodes. The returned `id` cannot be used in |
| /// [`CreateEdge`]. Edges must target a specific input or output slot as |
| /// described above. |
| /// |
| /// + `options` Extensible options for creating the node. |
| /// - `id` ID of the newly-created node. Guaranteed to be unique. |
| /// - `node_properties` Additional properties of the newly-created node. |
| /// * error Reason the node could not be created. |
| CreateCustom(resource struct { |
| options CreateCustomOptions; |
| }) -> (resource struct { |
| id NodeId; |
| node_properties CustomNodeProperties; |
| }) error CreateNodeError; |
| |
| /// Deletes the given node. |
| /// |
| /// The node must exist. `DeleteNode(x)` will delete all incoming and |
| /// outgoing edges of node `x` before deleting the node itself. Once the node |
| /// has been deleted it cannot be mentioned by any future method calls. |
| /// |
| /// ## Parameters |
| /// |
| /// + `id` ID of the node to delete. |
| /// * error Reason the node could not be deleted. |
| DeleteNode(struct { |
| id NodeId; |
| }) -> (struct {}) error DeleteNodeError; |
| |
| /// Creates an edge from the source node to the destination node. |
| /// |
| /// Both nodes must exist. The source's output encoding must be supported by |
| /// the destination node. With the exception of Mixer nodes, most nodes |
| /// require inputs to have a specific (fixed) encoding. |
| /// |
| /// The source's output streams must use the same reference clock as the |
| /// destinations input stream, unless the destination is a Mixer, in which |
| /// case the source's output stream can use any clock. |
| /// |
| /// If the source's direction is `INPUT`, the dest's direction cannot be |
| /// `OUTPUT`. See [`PipelineDirection`] for additional discussion. |
| /// |
| /// ## Parameters |
| /// |
| /// + `dest_id` ID of the destination node. |
| /// + `source_id` ID of the source node. |
| /// + `options` Extensible options for creating the edge. |
| /// * error Reason the edge could not be created. |
| CreateEdge(resource struct { |
| dest_id NodeId; |
| source_id NodeId; |
| options CreateEdgeOptions; |
| }) -> (struct {}) error CreateEdgeError; |
| |
| /// Deletes the edge connecting the source node to the destination node. |
| /// |
| /// The edge must exist. |
| /// |
| /// + `dest_id` ID of the destination node. |
| /// + `source_id` ID of the source node. |
| /// * error Reason the edge could not be deleted. |
| DeleteEdge(struct { |
| dest_id NodeId; |
| source_id NodeId; |
| }) -> (struct {}) error DeleteEdgeError; |
| |
| /// Creates a thread. |
| /// |
| /// Each `CreateThread` call creates a new thread in the mixer service. This |
| /// new thread will be used to process audio for all Consumer nodes assigned |
| /// to this thread. |
| /// |
| /// For more details, see "Execution Model" under the description for |
| /// [`Graph`]. |
| /// |
| /// ## Parameters |
| /// |
| /// + `options` Extensible options for creating the thread. |
| /// - `id` ID of the newly-created thread. Guaranteed to be unique. |
| /// * error Reason the thread could not be created. |
| CreateThread(resource struct { |
| options CreateThreadOptions; |
| }) -> (struct { |
| id ThreadId; |
| }) error CreateThreadError; |
| |
| /// Deletes the given thread. |
| /// |
| /// The thread must exist. A thread cannot be deleted until all Consumer |
| /// nodes assigned to that thread have been deleted. |
| /// |
| /// + `id` ID of the thread to delete. |
| /// * error Reason the thread could not be deleted. |
| DeleteThread(struct { |
| id ThreadId; |
| }) -> (struct {}) error DeleteThreadError; |
| |
| /// Creates a GainStage. |
| /// |
| /// A GainStage controls gain that should be applied to an audio stream. |
| /// GainStages can be attached to edges going into and out of a Mixer node. |
| /// Gain can be set to a specific value (e.g. in decibels) or it can be |
| /// muted. |
| /// |
| /// By default, a GainStage applies no gain and is not muted. |
| /// |
| /// + `options` Extensible options for creating the GainStage. |
| /// - `id` ID of the newly-created GainStage. Guaranteed to be unique. |
| /// * error Reason the GainStage could not be created. |
| CreateGainStage(resource struct { |
| options CreateGainStageOptions; |
| }) -> (struct { |
| id GainStageId; |
| }) error CreateGainStageError; |
| |
| /// Deletes the given GainStage. |
| /// |
| /// The GainStage must exist. A GainStage cannot be deleted until all |
| /// associated edges have been deleted. |
| /// |
| /// + `id` ID of the GainStage to delete. |
| /// * error Reason the GainStage could not be deleted. |
| DeleteGainStage(struct { |
| id GainStageId; |
| }) -> (struct {}) error DeleteGainStageError; |
| |
| /// Creates a graph-controlled reference clock. |
| /// |
| /// The returned clock has `ZX_RIGHT_READ` and `ZX_RIGHT_DUPLICATE` but not |
| /// `ZX_RIGHT_WRITE`. The clock may be duplicated and used wherever a |
| /// reference clock is needed. The graph will rate change this clock until |
| /// the clock (or a duplicate of the clock) is passed to |
| /// [`ForgetGraphControlledReferenceClock`]. |
| /// |
| /// For more details, see "Clocks" under the description for [`Graph`]. |
| /// |
| /// - `reference_clock` The new clock |
| /// * error Error from `zx_clock_create` |
| CreateGraphControlledReferenceClock() -> (resource struct { |
| reference_clock zx.handle:CLOCK; |
| }) error zx.status; |
| |
| /// Forgets about a graph-controlled reference clock. |
| /// |
| /// The clock must have been previously created by |
| /// [`CreateGraphControlledReferenceClock`]. The clock will not be deleted (other |
| /// duplicates will still exist) however it will no longer be adjusted by |
| /// this Graph (or by anyone else, since this Graph was the only entity with |
| /// `ZX_RIGHT_WRITE`). |
| /// |
| /// + `reference_clock` The clock to forget |
| /// * error `ZX_ERR_NOT_FOUND` if `reference_clock` was not created by |
| /// [`CreateGraphControlledReferenceClock`] or if was already forgotten |
| ForgetGraphControlledReferenceClock(resource struct { |
| reference_clock zx.handle:CLOCK; |
| }) -> (struct {}) error zx.status; |
| |
| // TODO(fxbug.dev/87651): Start/Stop methods to control Producer and Consumer nodes |
| // TODO(fxbug.dev/87651): Watch method to observe delay (aka lead time) of a node |
| }; |
| |
| /// Type of errors return by [`CreateGraph`]. |
| type CreateGraphError = flexible enum {}; |
| |
| /// Type of errors return by CreateNode methods. |
| type CreateNodeError = flexible enum {}; |
| |
| /// Type of errors return by [`DeleteNode`]. |
| type DeleteNodeError = flexible enum { |
| /// The given `id` is invalid. |
| DOES_NOT_EXIST = 1; |
| }; |
| |
| /// Type of errors return by [`CreateEdge`]. |
| type CreateEdgeError = flexible enum { |
| /// The given `dest_id` is invalid. |
| INVALID_DEST_ID = 1; |
| |
| /// The dest does not support an additional input. |
| DEST_HAS_TOO_MANY_INPUTS = 2; |
| |
| /// The given `source_id` is invalid. |
| INVALID_SOURCE_ID = 3; |
| |
| /// The source does not support an additional output. |
| SOURCE_HAS_TOO_MANY_OUTPUTS = 4; |
| |
| /// The source's output is not compatible with the dest. |
| INCOMPATIBLE_FORMATS = 5; |
| |
| /// The source and dest are already connected. |
| ALREADY_CONNECTED = 6; |
| |
| /// This edge would create a cycle. |
| CYCLE = 7; |
| }; |
| |
| /// Type of errors return by [`DeleteEdge`]. |
| type DeleteEdgeError = flexible enum { |
| /// The given `dest_id` is invalid. |
| INVALID_DEST_ID = 1; |
| |
| /// The given `source_id` is invalid. |
| INVALID_SOURCE_ID = 2; |
| |
| /// The edge does not exist. |
| EDGE_NOT_FOUND = 3; |
| }; |
| |
| /// Type of errors return by [`CreateThread`]. |
| type CreateThreadError = flexible enum {}; |
| |
| /// Type of errors return by [`DeleteThread`]. |
| type DeleteThreadError = flexible enum { |
| /// The given `id` is invalid. |
| INVALID_ID = 1; |
| |
| /// There are still Consumer nodes assigned to this thread. These Consumers |
| /// must be deleted before the Thread can be deleted. |
| STILL_IN_USE = 2; |
| }; |
| |
| /// Type of errors return by [`CreateGainStage`]. |
| type CreateGainStageError = flexible enum {}; |
| |
| /// Type of errors return by [`DeleteGainStage`]. |
| type DeleteGainStageError = flexible enum { |
| /// The given `id` is invalid. |
| INVALID_ID = 1; |
| |
| /// There are still edges using this GainStage. These edges must be deleted |
| /// before the GainStage can be deleted. |
| STILL_IN_USE = 2; |
| }; |