Source code
Revision control
Copy as Markdown
Other Tools
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
package org.mozilla.gecko.media;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaCodecInfo.VideoCapabilities;
import android.media.MediaCodecList;
import android.media.MediaCrypto;
import android.media.MediaFormat;
import android.os.IBinder;
import android.os.RemoteException;
import android.util.Log;
import android.view.Surface;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import org.mozilla.gecko.gfx.GeckoSurface;
/* package */ final class Codec extends ICodec.Stub implements IBinder.DeathRecipient {
private static final String LOGTAG = "GeckoRemoteCodec";
private static final boolean DEBUG = false;
public static final String SW_CODEC_PREFIX = "OMX.google.";
public enum Error {
DECODE,
FATAL
}
private final class Callbacks implements AsyncCodec.Callbacks {
@Override
public void onInputBufferAvailable(final AsyncCodec codec, final int index) {
mInputProcessor.onBuffer(index);
}
@Override
public void onOutputBufferAvailable(
final AsyncCodec codec, final int index, final MediaCodec.BufferInfo info) {
mOutputProcessor.onBuffer(index, info);
}
@Override
public void onError(final AsyncCodec codec, final int error) {
reportError(Error.FATAL, new Exception("codec error:" + error));
}
@Override
public void onOutputFormatChanged(final AsyncCodec codec, final MediaFormat format) {
mOutputProcessor.onFormatChanged(format);
}
}
private static final class Input {
public final Sample sample;
public boolean reported;
public Input(final Sample sample) {
this.sample = sample;
}
}
private final class InputProcessor {
private boolean mHasInputCapacitySet;
private Queue<Integer> mAvailableInputBuffers = new LinkedList<>();
private Queue<Sample> mDequeuedSamples = new LinkedList<>();
private Queue<Input> mInputSamples = new LinkedList<>();
private boolean mStopped;
private synchronized Sample onAllocate(final int size) {
final Sample sample = mSamplePool.obtainInput(size);
sample.session = mSession;
mDequeuedSamples.add(sample);
return sample;
}
private synchronized void onSample(final Sample sample) {
if (sample == null) {
// Ignore empty input.
mSamplePool.recycleInput(mDequeuedSamples.remove());
Log.w(LOGTAG, "WARN: empty input sample");
return;
}
if (sample.isEOS()) {
queueSample(sample);
return;
}
if (sample.session >= mSession) {
final Sample dequeued = mDequeuedSamples.remove();
dequeued.setBufferInfo(sample.info);
dequeued.setCryptoInfo(sample.cryptoInfo);
queueSample(dequeued);
}
sample.dispose();
}
private void queueSample(final Sample sample) {
if (!mInputSamples.offer(new Input(sample))) {
reportError(Error.FATAL, new Exception("FAIL: input sample queue is full"));
return;
}
try {
feedSampleToBuffer();
} catch (final Exception e) {
reportError(Error.FATAL, e);
}
}
private synchronized void onBuffer(final int index) {
if (mStopped || !isValidBuffer(index)) {
return;
}
if (!mHasInputCapacitySet) {
final int capacity = mCodec.getInputBuffer(index).capacity();
if (capacity > 0) {
mSamplePool.setInputBufferSize(capacity);
mHasInputCapacitySet = true;
}
}
if (mAvailableInputBuffers.offer(index)) {
feedSampleToBuffer();
} else {
reportError(Error.FATAL, new Exception("FAIL: input buffer queue is full"));
}
}
private boolean isValidBuffer(final int index) {
try {
return mCodec.getInputBuffer(index) != null;
} catch (final IllegalStateException e) {
if (DEBUG) {
Log.d(LOGTAG, "invalid input buffer#" + index, e);
}
return false;
}
}
private void feedSampleToBuffer() {
while (!mAvailableInputBuffers.isEmpty() && !mInputSamples.isEmpty()) {
final int index = mAvailableInputBuffers.poll();
if (!isValidBuffer(index)) {
continue;
}
int len = 0;
final Sample sample = mInputSamples.poll().sample;
final long pts = sample.info.presentationTimeUs;
final int flags = sample.info.flags;
final MediaCodec.CryptoInfo cryptoInfo = sample.cryptoInfo;
if (!sample.isEOS() && sample.bufferId != Sample.NO_BUFFER) {
len = sample.info.size;
final ByteBuffer buf = mCodec.getInputBuffer(index);
try {
mSamplePool
.getInputBuffer(sample.bufferId)
.writeToByteBuffer(buf, sample.info.offset, len);
} catch (final IOException e) {
e.printStackTrace();
len = 0;
}
mSamplePool.recycleInput(sample);
}
try {
if (cryptoInfo != null && len > 0) {
mCodec.queueSecureInputBuffer(index, 0, cryptoInfo, pts, flags);
} else {
mCodec.queueInputBuffer(index, 0, len, pts, flags);
}
mCallbacks.onInputQueued(pts);
} catch (final RemoteException e) {
e.printStackTrace();
} catch (final Exception e) {
reportError(Error.FATAL, e);
return;
}
}
reportPendingInputs();
}
private void reportPendingInputs() {
try {
for (final Input i : mInputSamples) {
if (!i.reported) {
i.reported = true;
mCallbacks.onInputPending(i.sample.info.presentationTimeUs);
}
}
} catch (final RemoteException e) {
e.printStackTrace();
}
}
private synchronized void reset() {
for (final Input i : mInputSamples) {
if (!i.sample.isEOS()) {
mSamplePool.recycleInput(i.sample);
}
}
mInputSamples.clear();
for (final Sample s : mDequeuedSamples) {
mSamplePool.recycleInput(s);
}
mDequeuedSamples.clear();
mAvailableInputBuffers.clear();
}
private synchronized void start() {
if (!mStopped) {
return;
}
mStopped = false;
}
private synchronized void stop() {
if (mStopped) {
return;
}
mStopped = true;
reset();
}
}
private static final class Output {
public final Sample sample;
public final int index;
public Output(final Sample sample, final int index) {
this.sample = sample;
this.index = index;
}
}
private class OutputProcessor {
private final boolean mRenderToSurface;
private boolean mHasOutputCapacitySet;
private Queue<Output> mSentOutputs = new LinkedList<>();
private boolean mStopped;
private OutputProcessor(final boolean renderToSurface) {
mRenderToSurface = renderToSurface;
}
private synchronized void onBuffer(final int index, final MediaCodec.BufferInfo info) {
if (mStopped || !isValidBuffer(index)) {
return;
}
try {
final Sample output = obtainOutputSample(index, info);
mSentOutputs.add(new Output(output, index));
output.session = mSession;
mCallbacks.onOutput(output);
} catch (final Exception e) {
e.printStackTrace();
mCodec.releaseOutputBuffer(index, false);
}
final boolean eos = (info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0;
if (DEBUG && eos) {
Log.d(LOGTAG, "output EOS");
}
}
private boolean isValidBuffer(final int index) {
try {
return (mCodec.getOutputBuffer(index) != null) || mRenderToSurface;
} catch (final IllegalStateException e) {
if (DEBUG) {
Log.e(LOGTAG, "invalid buffer#" + index, e);
}
return false;
}
}
private Sample obtainOutputSample(final int index, final MediaCodec.BufferInfo info) {
final Sample sample = mSamplePool.obtainOutput(info);
if (mRenderToSurface) {
return sample;
}
final ByteBuffer output = mCodec.getOutputBuffer(index);
if (!mHasOutputCapacitySet) {
final int capacity = output.capacity();
if (capacity > 0) {
mSamplePool.setOutputBufferSize(capacity);
mHasOutputCapacitySet = true;
}
}
if (info.size > 0) {
try {
mSamplePool
.getOutputBuffer(sample.bufferId)
.readFromByteBuffer(output, info.offset, info.size);
} catch (final IOException e) {
Log.e(LOGTAG, "Fail to read output buffer:" + e.getMessage());
}
}
return sample;
}
private synchronized void onRelease(final Sample sample, final boolean render) {
final Output output = mSentOutputs.poll();
if (output != null) {
mCodec.releaseOutputBuffer(output.index, render);
mSamplePool.recycleOutput(output.sample);
} else if (DEBUG) {
Log.d(LOGTAG, sample + " already released");
}
sample.dispose();
}
private synchronized void onFormatChanged(final MediaFormat format) {
if (mStopped) {
return;
}
try {
mCallbacks.onOutputFormatChanged(new FormatParam(format));
} catch (final RemoteException re) {
// Dead recipient.
re.printStackTrace();
}
}
private synchronized void reset() {
for (final Output o : mSentOutputs) {
mCodec.releaseOutputBuffer(o.index, false);
mSamplePool.recycleOutput(o.sample);
}
mSentOutputs.clear();
}
private synchronized void start() {
if (!mStopped) {
return;
}
mStopped = false;
}
private synchronized void stop() {
if (mStopped) {
return;
}
mStopped = true;
reset();
}
}
private volatile ICodecCallbacks mCallbacks;
private GeckoSurface mSurface;
private AsyncCodec mCodec;
private InputProcessor mInputProcessor;
private OutputProcessor mOutputProcessor;
private long mSession;
private SamplePool mSamplePool;
// Values will be updated after configure called.
private volatile boolean mIsAdaptivePlaybackSupported = false;
private volatile boolean mIsHardwareAccelerated = false;
private boolean mIsTunneledPlaybackSupported = false;
public synchronized void setCallbacks(final ICodecCallbacks callbacks) throws RemoteException {
mCallbacks = callbacks;
callbacks.asBinder().linkToDeath(this, 0);
}
// IBinder.DeathRecipient
@Override
public synchronized void binderDied() {
Log.e(LOGTAG, "Callbacks is dead");
try {
release();
} catch (final RemoteException e) {
// Nowhere to report the error.
}
}
@Override
public synchronized boolean configure(
final FormatParam format, final GeckoSurface surface, final int flags, final String drmStubId)
throws RemoteException {
if (mCallbacks == null) {
Log.e(LOGTAG, "FAIL: callbacks must be set before calling configure()");
return false;
}
if (mCodec != null) {
if (DEBUG) {
Log.d(LOGTAG, "release existing codec: " + mCodec);
}
mCodec.release();
}
if (DEBUG) {
Log.d(LOGTAG, "configure " + this);
}
final MediaFormat fmt = format.asFormat();
final String mime = fmt.getString(MediaFormat.KEY_MIME);
if (mime == null || mime.isEmpty()) {
Log.e(LOGTAG, "invalid MIME type: " + mime);
return false;
}
final List<String> found =
findMatchingCodecNames(fmt, flags == MediaCodec.CONFIGURE_FLAG_ENCODE);
for (final String name : found) {
final AsyncCodec codec =
configureCodec(
name, fmt, surface != null ? surface.getSurface() : null, flags, drmStubId);
if (codec == null) {
Log.w(LOGTAG, "unable to configure " + name + ". Try next.");
continue;
}
mIsHardwareAccelerated = !name.startsWith(SW_CODEC_PREFIX);
mCodec = codec;
if (flags == MediaCodec.CONFIGURE_FLAG_ENCODE && fmt.containsKey(MediaFormat.KEY_WIDTH)) {
final MediaFormat inputFormat = mCodec.getInputFormat();
if (inputFormat != null) {
if (inputFormat.containsKey(MediaFormat.KEY_STRIDE)) {
fmt.setInteger(MediaFormat.KEY_STRIDE, inputFormat.getInteger(MediaFormat.KEY_STRIDE));
}
if (inputFormat.containsKey(MediaFormat.KEY_SLICE_HEIGHT)) {
fmt.setInteger(
MediaFormat.KEY_SLICE_HEIGHT, inputFormat.getInteger(MediaFormat.KEY_SLICE_HEIGHT));
}
}
}
mInputProcessor = new InputProcessor();
final boolean renderToSurface = surface != null;
mOutputProcessor = new OutputProcessor(renderToSurface);
mSamplePool = new SamplePool(name, renderToSurface);
if (renderToSurface) {
mIsTunneledPlaybackSupported = mCodec.isTunneledPlaybackSupported(mime);
mSurface = surface; // Take ownership of surface.
}
if (DEBUG) {
Log.d(LOGTAG, codec.toString() + " created. Render to surface?" + renderToSurface);
}
return true;
}
return false;
}
private List<String> findMatchingCodecNames(final MediaFormat format, final boolean isEncoder) {
final String mimeType = format.getString(MediaFormat.KEY_MIME);
// Missing width and height value in format means audio;
// Video format should never has 0 width or height.
final int width =
format.containsKey(MediaFormat.KEY_WIDTH) ? format.getInteger(MediaFormat.KEY_WIDTH) : 0;
final int height =
format.containsKey(MediaFormat.KEY_HEIGHT) ? format.getInteger(MediaFormat.KEY_HEIGHT) : 0;
int numCodecs = 0;
final List<String> found = new ArrayList<>();
try {
numCodecs = MediaCodecList.getCodecCount();
} catch (final RuntimeException e) {
Log.e(LOGTAG, "Failed retrieving codec count finding matching codec names", e);
return found;
}
for (int i = 0; i < numCodecs; i++) {
final MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i);
if (info.isEncoder() == !isEncoder) {
continue;
}
final String[] types = info.getSupportedTypes();
for (final String t : types) {
if (!t.equalsIgnoreCase(mimeType)) {
continue;
}
final String name = info.getName();
// API 21+ provide a method to query whether supplied size is supported. For
// older version, just avoid software video encoders.
if (isEncoder && width > 0 && height > 0) {
final VideoCapabilities c = info.getCapabilitiesForType(mimeType).getVideoCapabilities();
if (c != null && !c.isSizeSupported(width, height)) {
if (DEBUG) {
Log.d(LOGTAG, name + ": " + width + "x" + height + " not supported");
}
continue;
}
}
found.add(name);
if (DEBUG) {
Log.d(
LOGTAG,
"found " + (isEncoder ? "encoder:" : "decoder:") + name + " for mime:" + mimeType);
}
}
}
return found;
}
private AsyncCodec configureCodec(
final String name,
final MediaFormat format,
final Surface surface,
final int flags,
final String drmStubId) {
try {
final AsyncCodec codec = AsyncCodecFactory.create(name);
codec.setCallbacks(new Callbacks(), null);
final MediaCrypto crypto = RemoteMediaDrmBridgeStub.getMediaCrypto(drmStubId);
if (DEBUG) {
Log.d(
LOGTAG,
"configure mediacodec with crypto(" + (crypto != null) + ") / Id :" + drmStubId);
}
if (surface != null) {
setupAdaptivePlayback(codec, format);
}
codec.configure(format, surface, crypto, flags);
return codec;
} catch (final Exception e) {
Log.e(LOGTAG, "codec creation error", e);
return null;
}
}
private void setupAdaptivePlayback(final AsyncCodec codec, final MediaFormat format) {
// Video decoder should config with adaptive playback capability.
mIsAdaptivePlaybackSupported =
codec.isAdaptivePlaybackSupported(format.getString(MediaFormat.KEY_MIME));
if (mIsAdaptivePlaybackSupported) {
if (DEBUG) {
Log.d(LOGTAG, "codec supports adaptive playback = " + mIsAdaptivePlaybackSupported);
}
// TODO: may need to find a way to not use hard code to decide the max w/h.
format.setInteger(MediaFormat.KEY_MAX_WIDTH, 1920);
format.setInteger(MediaFormat.KEY_MAX_HEIGHT, 1080);
}
}
@Override
public synchronized boolean isAdaptivePlaybackSupported() {
return mIsAdaptivePlaybackSupported;
}
@Override
public synchronized boolean isHardwareAccelerated() {
return mIsHardwareAccelerated;
}
@Override
public synchronized boolean isTunneledPlaybackSupported() {
return mIsTunneledPlaybackSupported;
}
@Override
public synchronized void start() throws RemoteException {
if (DEBUG) {
Log.d(LOGTAG, "start " + this);
}
mInputProcessor.start();
mOutputProcessor.start();
try {
mCodec.start();
} catch (final Exception e) {
reportError(Error.FATAL, e);
}
}
private void reportError(final Error error, final Exception e) {
if (e != null) {
e.printStackTrace();
}
try {
mCallbacks.onError(error == Error.FATAL);
} catch (final NullPointerException ne) {
// mCallbacks has been disposed by release().
} catch (final RemoteException re) {
re.printStackTrace();
}
}
@Override
public synchronized void stop() throws RemoteException {
if (DEBUG) {
Log.d(LOGTAG, "stop " + this);
}
try {
mInputProcessor.stop();
mOutputProcessor.stop();
mCodec.stop();
} catch (final Exception e) {
reportError(Error.FATAL, e);
}
}
@Override
public synchronized void flush() throws RemoteException {
if (DEBUG) {
Log.d(LOGTAG, "flush " + this);
}
try {
mInputProcessor.stop();
mOutputProcessor.stop();
mCodec.flush();
if (DEBUG) {
Log.d(LOGTAG, "flushed " + this);
}
mInputProcessor.start();
mOutputProcessor.start();
mCodec.resumeReceivingInputs();
mSession++;
} catch (final Exception e) {
reportError(Error.FATAL, e);
}
}
@Override
public synchronized Sample dequeueInput(final int size) throws RemoteException {
try {
return mInputProcessor.onAllocate(size);
} catch (final Exception e) {
// Translate allocation error to remote exception.
throw new RemoteException(e.getMessage());
}
}
@Override
public synchronized SampleBuffer getInputBuffer(final int id) {
if (mSamplePool == null) {
return null;
}
return mSamplePool.getInputBuffer(id);
}
@Override
public synchronized SampleBuffer getOutputBuffer(final int id) {
if (mSamplePool == null) {
return null;
}
return mSamplePool.getOutputBuffer(id);
}
@Override
public synchronized void queueInput(final Sample sample) throws RemoteException {
try {
mInputProcessor.onSample(sample);
} catch (final Exception e) {
throw new RemoteException(e.getMessage());
}
}
@Override
public synchronized void setBitrate(final int bps) {
try {
mCodec.setBitrate(bps);
} catch (final Exception e) {
reportError(Error.FATAL, e);
}
}
@Override
public synchronized void releaseOutput(final Sample sample, final boolean render) {
try {
mOutputProcessor.onRelease(sample, render);
} catch (final Exception e) {
reportError(Error.FATAL, e);
}
}
@Override
public synchronized void release() throws RemoteException {
if (DEBUG) {
Log.d(LOGTAG, "release " + this);
}
try {
// In case Codec.stop() is not called yet.
mInputProcessor.stop();
mOutputProcessor.stop();
mCodec.release();
} catch (final Exception e) {
reportError(Error.FATAL, e);
}
mCodec = null;
mSamplePool.reset();
mSamplePool = null;
mCallbacks.asBinder().unlinkToDeath(this, 0);
mCallbacks = null;
if (mSurface != null) {
mSurface.release();
mSurface = null;
}
}
}