audio_device_ios.h

/*

 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.

 *  Use of this source code is governed by a BSD-style license

 *  that can be found in the LICENSE file in the root of the source

 *  tree. An additional intellectual property rights grant can be found

 *  in the file PATENTS.  All contributing project authors may

 *  be found in the AUTHORS file in the root of the source tree.

*/

#ifndef SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_

#define SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_

#include <atomic>

#include <memory>

#include "api/scoped_refptr.h"

#include "api/sequence_checker.h"

#include "api/task_queue/pending_task_safety_flag.h"

#include "audio_session_observer.h"

#include "modules/audio_device/audio_device_generic.h"

#include "rtc_base/buffer.h"

#include "rtc_base/thread.h"

#include "rtc_base/thread_annotations.h"

#include "sdk/objc/base/RTCMacros.h"

#include "voice_processing_audio_unit.h"

RTC_FWD_DECL_OBJC_CLASS(RTCNativeAudioSessionDelegateAdapter);

namespace webrtc {

class FineAudioBuffer;

namespace ios_adm {

// Implements full duplex 16-bit mono PCM audio support for iOS using a

// Voice-Processing (VP) I/O audio unit in Core Audio. The VP I/O audio unit

// supports audio echo cancellation. It also adds automatic gain control,

// adjustment of voice-processing quality and muting.

//

// An instance must be created and destroyed on one and the same thread.

// All supported public methods must also be called on the same thread.

// A thread checker will RTC_DCHECK if any supported method is called on an

// invalid thread.

//

// Recorded audio will be delivered on a real-time internal I/O thread in the

// audio unit. The audio unit will also ask for audio data to play out on this

// same thread.

class AudioDeviceIOS : public AudioDeviceGeneric,

                       public AudioSessionObserver,

                       public VoiceProcessingAudioUnitObserver {

 public:

  explicit AudioDeviceIOS(

      bool bypass_voice_processing,

      AudioDeviceModule::MutedSpeechEventHandler muted_speech_event_handler);

  ~AudioDeviceIOS() override;

  void AttachAudioBuffer(AudioDeviceBuffer* audioBuffer) override;

  InitStatus Init() override;

  int32_t Terminate() override;

  bool Initialized() const override;

  int32_t InitPlayout() override;

  bool PlayoutIsInitialized() const override;

  int32_t InitRecording() override;

  bool RecordingIsInitialized() const override;

  int32_t StartPlayout() override;

  int32_t StopPlayout() override;

  bool Playing() const override;

  int32_t StartRecording() override;

  int32_t StopRecording() override;

  bool Recording() const override;

  // These methods returns hard-coded delay values and not dynamic delay

  // estimates. The reason is that iOS supports a built-in AEC and the WebRTC

  // AEC will always be disabled in the Libjingle layer to avoid running two

  // AEC implementations at the same time. And, it saves resources to avoid

  // updating these delay values continuously.

  // TODO(henrika): it would be possible to mark these two methods as not

  // implemented since they are only called for A/V-sync purposes today and

  // A/V-sync is not supported on iOS. However, we avoid adding error messages

  // the log by using these dummy implementations instead.

  int32_t PlayoutDelay(uint16_t& delayMS) const override;

  // No implementation for playout underrun on iOS. We override it to avoid a

  // periodic log that it isn't available from the base class.

  int32_t GetPlayoutUnderrunCount() const override { return -1; }

  // Native audio parameters stored during construction.

  // These methods are unique for the iOS implementation.

  int GetPlayoutAudioParameters(AudioParameters* params) const override;

  int GetRecordAudioParameters(AudioParameters* params) const override;

  // These methods are currently not fully implemented on iOS:

  // See audio_device_not_implemented.cc for trivial implementations.

  int32_t ActiveAudioLayer(

      AudioDeviceModule::AudioLayer& audioLayer) const override;

  int32_t PlayoutIsAvailable(bool& available) override;

  int32_t RecordingIsAvailable(bool& available) override;

  int16_t PlayoutDevices() override;

  int16_t RecordingDevices() override;

  int32_t PlayoutDeviceName(uint16_t index,

                            char name[kAdmMaxDeviceNameSize],

                            char guid[kAdmMaxGuidSize]) override;

  int32_t RecordingDeviceName(uint16_t index,

                              char name[kAdmMaxDeviceNameSize],

                              char guid[kAdmMaxGuidSize]) override;

  int32_t SetPlayoutDevice(uint16_t index) override;

  int32_t SetPlayoutDevice(

      AudioDeviceModule::WindowsDeviceType device) override;

  int32_t SetRecordingDevice(uint16_t index) override;

  int32_t SetRecordingDevice(

      AudioDeviceModule::WindowsDeviceType device) override;

  int32_t InitSpeaker() override;

  bool SpeakerIsInitialized() const override;

  int32_t InitMicrophone() override;

  bool MicrophoneIsInitialized() const override;

  int32_t SpeakerVolumeIsAvailable(bool& available) override;

  int32_t SetSpeakerVolume(uint32_t volume) override;

  int32_t SpeakerVolume(uint32_t& volume) const override;

  int32_t MaxSpeakerVolume(uint32_t& maxVolume) const override;

  int32_t MinSpeakerVolume(uint32_t& minVolume) const override;

  int32_t MicrophoneVolumeIsAvailable(bool& available) override;

  int32_t SetMicrophoneVolume(uint32_t volume) override;

  int32_t MicrophoneVolume(uint32_t& volume) const override;

  int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const override;

  int32_t MinMicrophoneVolume(uint32_t& minVolume) const override;

  int32_t MicrophoneMuteIsAvailable(bool& available) override;

  int32_t SetMicrophoneMute(bool enable) override;

  int32_t MicrophoneMute(bool& enabled) const override;

  int32_t SpeakerMuteIsAvailable(bool& available) override;

  int32_t SetSpeakerMute(bool enable) override;

  int32_t SpeakerMute(bool& enabled) const override;

  int32_t StereoPlayoutIsAvailable(bool& available) override;

  int32_t SetStereoPlayout(bool enable) override;

  int32_t StereoPlayout(bool& enabled) const override;

  int32_t StereoRecordingIsAvailable(bool& available) override;

  int32_t SetStereoRecording(bool enable) override;

  int32_t StereoRecording(bool& enabled) const override;

  // AudioSessionObserver methods. May be called from any thread.

  void OnInterruptionBegin() override;

  void OnInterruptionEnd() override;

  void OnValidRouteChange() override;

  void OnCanPlayOrRecordChange(bool can_play_or_record) override;

  void OnChangedOutputVolume() override;

  // VoiceProcessingAudioUnitObserver methods.

  OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,

                                 const AudioTimeStamp* time_stamp,

                                 UInt32 bus_number,

                                 UInt32 num_frames,

                                 AudioBufferList* io_data) override;

  OSStatus OnGetPlayoutData(AudioUnitRenderActionFlags* flags,

                            const AudioTimeStamp* time_stamp,

                            UInt32 bus_number,

                            UInt32 num_frames,

                            AudioBufferList* io_data) override;

  void OnReceivedMutedSpeechActivity(

      AUVoiceIOSpeechActivityEvent event) override;

  bool IsInterrupted();

 private:

  // Called by the relevant AudioSessionObserver methods on `thread_`.

  void HandleInterruptionBegin();

  void HandleInterruptionEnd();

  void HandleValidRouteChange();

  void HandleCanPlayOrRecordChange(bool can_play_or_record);

  void HandleSampleRateChange();

  void HandlePlayoutGlitchDetected();

  void HandleOutputVolumeChange();

  // Uses current `playout_parameters_` and `record_parameters_` to inform the

  // audio device buffer (ADB) about our internal audio parameters.

  void UpdateAudioDeviceBuffer();

  // Since the preferred audio parameters are only hints to the OS, the actual

  // values may be different once the AVAudioSession has been activated.

  // This method asks for the current hardware parameters and takes actions

  // if they should differ from what we have asked for initially. It also

  // defines `playout_parameters_` and `record_parameters_`.

  void SetupAudioBuffersForActiveAudioSession();

  // Creates the audio unit.

  bool CreateAudioUnit();

  // Updates the audio unit state based on current state.

  void UpdateAudioUnit(bool can_play_or_record);

  // Configures the audio session for WebRTC.

  bool ConfigureAudioSession();

  // Like above, but requires caller to already hold session lock.

  bool ConfigureAudioSessionLocked();

  // Unconfigures the audio session.

  void UnconfigureAudioSession();

  // Activates our audio session, creates and initializes the voice-processing

  // audio unit and verifies that we got the preferred native audio parameters.

  bool InitPlayOrRecord();

  // Closes and deletes the voice-processing I/O unit.

  void ShutdownPlayOrRecord();

  // Resets thread-checkers before a call is restarted.

  void PrepareForNewStart();

  // Determines whether voice processing should be enabled or disabled.

  const bool bypass_voice_processing_;

  // Handle a user speaking during muted event

  AudioDeviceModule::MutedSpeechEventHandler muted_speech_event_handler_;

  // Native I/O audio thread checker.

  SequenceChecker io_thread_checker_;

  // Thread that this object is created on.

  rtc::Thread* thread_;

  // Raw pointer handle provided to us in AttachAudioBuffer(). Owned by the

  // AudioDeviceModuleImpl class and called by AudioDeviceModule::Create().

  // The AudioDeviceBuffer is a member of the AudioDeviceModuleImpl instance

  // and therefore outlives this object.

  AudioDeviceBuffer* audio_device_buffer_;

  // Contains audio parameters (sample rate, #channels, buffer size etc.) for

  // the playout and recording sides. These structure is set in two steps:

  // first, native sample rate and #channels are defined in Init(). Next, the

  // audio session is activated and we verify that the preferred parameters

  // were granted by the OS. At this stage it is also possible to add a third

  // component to the parameters; the native I/O buffer duration.

  // A RTC_CHECK will be hit if we for some reason fail to open an audio session

  // using the specified parameters.

  AudioParameters playout_parameters_;

  AudioParameters record_parameters_;

  // The AudioUnit used to play and record audio.

  std::unique_ptr<VoiceProcessingAudioUnit> audio_unit_;

  // FineAudioBuffer takes an AudioDeviceBuffer which delivers audio data

  // in chunks of 10ms. It then allows for this data to be pulled in

  // a finer or coarser granularity. I.e. interacting with this class instead

  // of directly with the AudioDeviceBuffer one can ask for any number of

  // audio data samples. Is also supports a similar scheme for the recording

  // side.

  // Example: native buffer size can be 128 audio frames at 16kHz sample rate.

  // WebRTC will provide 480 audio frames per 10ms but iOS asks for 128

  // in each callback (one every 8ms). This class can then ask for 128 and the

  // FineAudioBuffer will ask WebRTC for new data only when needed and also

  // cache non-utilized audio between callbacks. On the recording side, iOS

  // can provide audio data frames of size 128 and these are accumulated until

  // enough data to supply one 10ms call exists. This 10ms chunk is then sent

  // to WebRTC and the remaining part is stored.

  std::unique_ptr<FineAudioBuffer> fine_audio_buffer_;

  // Temporary storage for recorded data. AudioUnitRender() renders into this

  // array as soon as a frame of the desired buffer size has been recorded.

  // On real iOS devices, the size will be fixed and set once. For iOS

  // simulators, the size can vary from callback to callback and the size

  // will be changed dynamically to account for this behavior.

  rtc::BufferT<int16_t> record_audio_buffer_;

  // Set to 1 when recording is active and 0 otherwise.

  std::atomic<int> recording_;

  // Set to 1 when playout is active and 0 otherwise.

  std::atomic<int> playing_;

  // Set to true after successful call to Init(), false otherwise.

  bool initialized_ RTC_GUARDED_BY(thread_);

  // Set to true after successful call to InitRecording() or InitPlayout(),

  // false otherwise.

  bool audio_is_initialized_;

  // Set to true if audio session is interrupted, false otherwise.

  bool is_interrupted_;

  // Audio interruption observer instance.

  RTCNativeAudioSessionDelegateAdapter* audio_session_observer_

      RTC_GUARDED_BY(thread_);

  // Set to true if we've activated the audio session.

  bool has_configured_session_ RTC_GUARDED_BY(thread_);

  // Counts number of detected audio glitches on the playout side.

  int64_t num_detected_playout_glitches_ RTC_GUARDED_BY(thread_);

  int64_t last_playout_time_ RTC_GUARDED_BY(io_thread_checker_);

  // Counts number of playout callbacks per call.

  // The value is updated on the native I/O thread and later read on the

  // creating `thread_` but at this stage no audio is active.

  // Hence, it is a "thread safe" design and no lock is needed.

  int64_t num_playout_callbacks_;

  // Contains the time for when the last output volume change was detected.

  int64_t last_output_volume_change_time_ RTC_GUARDED_BY(thread_);

  // Avoids running pending task after `this` is Terminated.

  rtc::scoped_refptr<PendingTaskSafetyFlag> safety_ =

      PendingTaskSafetyFlag::Create();

  // Ratio between mach tick units and nanosecond. Used to change mach tick

  // units to nanoseconds.

  double machTickUnitsToNanoseconds_;

};

}  // namespace ios_adm

}  // namespace webrtc

#endif  // SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_

Source code

Revision control

Copy as Markdown

Other Tools