Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include "TouchResampler.h"
#include "nsAlgorithm.h"
/**
* TouchResampler implementation
*/
namespace mozilla {
namespace widget {
// The values below have been tested and found to be acceptable on a device
// with a display refresh rate of 60Hz and touch sampling rate of 100Hz.
// While their "ideal" values are dependent on the exact rates of each device,
// the values we've picked below should be somewhat robust across a variation of
// different rates. They mostly aim to avoid making predictions that are too far
// away (in terms of distance) from the finger, and to detect pauses in the
// finger motion without too much delay.
// Maximum time between two consecutive data points to consider resampling
// between them.
// Values between 1x and 5x of the touch sampling interval are reasonable.
static const double kTouchResampleWindowSize = 40.0;
// These next two values constrain the sampling timestamp.
// Our caller will usually adjust frame timestamps to be slightly in the past,
// for example by 5ms. This means that, during normal operation, we will
// maximally need to predict by [touch sampling rate] minus 5ms.
// So we would like kTouchResampleMaxPredictMs to satisfy the following:
// kTouchResampleMaxPredictMs + [frame time adjust] > [touch sampling rate]
static const double kTouchResampleMaxPredictMs = 8.0;
// This one is a protection against very outdated frame timestamps.
// Values larger than the touch sampling interval and less than 3x of the vsync
// interval are reasonable.
static const double kTouchResampleMaxBacksampleMs = 20.0;
// The maximum age of the most recent data point to consider resampling.
// Should be between 1x and 3x of the touch sampling interval.
static const double kTouchResampleOldTouchThresholdMs = 17.0;
uint64_t TouchResampler::ProcessEvent(MultiTouchInput&& aInput) {
mCurrentTouches.UpdateFromEvent(aInput);
uint64_t eventId = mNextEventId;
mNextEventId++;
if (aInput.mType == MultiTouchInput::MULTITOUCH_MOVE) {
// Touch move events are deferred until NotifyFrame.
mDeferredTouchMoveEvents.push({std::move(aInput), eventId});
} else {
// Non-move events are transferred to the outgoing queue unmodified.
// If there are pending touch move events, flush those out first, so that
// events are emitted in the right order.
FlushDeferredTouchMoveEventsUnresampled();
if (mInResampledState) {
// Return to a non-resampled state before emitting a non-move event.
ReturnToNonResampledState();
}
EmitEvent(std::move(aInput), eventId);
}
return eventId;
}
void TouchResampler::NotifyFrame(const TimeStamp& aTimeStamp) {
TimeStamp lastTouchTime = mCurrentTouches.LatestDataPointTime();
if (mDeferredTouchMoveEvents.empty() ||
(lastTouchTime &&
lastTouchTime < aTimeStamp - TimeDuration::FromMilliseconds(
kTouchResampleOldTouchThresholdMs))) {
// We haven't received a touch move event in a while, so the fingers must
// have stopped moving. Flush any old touch move events.
FlushDeferredTouchMoveEventsUnresampled();
if (mInResampledState) {
// Make sure we pause at the resting position that we actually observed,
// and not at a resampled position.
ReturnToNonResampledState();
}
// Clear touch location history so that we don't resample across a pause.
mCurrentTouches.ClearDataPoints();
return;
}
MOZ_RELEASE_ASSERT(lastTouchTime);
TimeStamp lowerBound = lastTouchTime - TimeDuration::FromMilliseconds(
kTouchResampleMaxBacksampleMs);
TimeStamp upperBound = lastTouchTime + TimeDuration::FromMilliseconds(
kTouchResampleMaxPredictMs);
TimeStamp sampleTime = clamped(aTimeStamp, lowerBound, upperBound);
if (mLastEmittedEventTime && sampleTime < mLastEmittedEventTime) {
// Keep emitted timestamps in order.
sampleTime = mLastEmittedEventTime;
}
// We have at least one pending touch move event. Pick one of the events from
// mDeferredTouchMoveEvents as the base event for the resampling adjustment.
// We want to produce an event stream whose timestamps are in the right order.
// As the base event, use the first event that's at or after sampleTime,
// unless there is no such event, in that case use the last one we have. We
// will set the timestamp on the resampled event to sampleTime later.
// Flush out any older events so that everything remains in the right order.
MultiTouchInput input;
uint64_t eventId;
while (true) {
MOZ_RELEASE_ASSERT(!mDeferredTouchMoveEvents.empty());
std::tie(input, eventId) = std::move(mDeferredTouchMoveEvents.front());
mDeferredTouchMoveEvents.pop();
if (mDeferredTouchMoveEvents.empty() || input.mTimeStamp >= sampleTime) {
break;
}
// Flush this event to the outgoing queue without resampling. What ends up
// on the screen will still be smooth because we will proceed to emit a
// resampled event before the paint for this frame starts.
PrependLeftoverHistoricalData(&input);
MOZ_RELEASE_ASSERT(input.mTimeStamp < sampleTime);
EmitEvent(std::move(input), eventId);
}
mOriginalOfResampledTouchMove = Nothing();
// Compute the resampled touch positions.
nsTArray<ScreenIntPoint> resampledPositions;
bool anyPositionDifferentFromOriginal = false;
for (const auto& touch : input.mTouches) {
ScreenIntPoint resampledPosition =
mCurrentTouches.ResampleTouchPositionAtTime(
touch.mIdentifier, touch.mScreenPoint, sampleTime);
if (resampledPosition != touch.mScreenPoint) {
anyPositionDifferentFromOriginal = true;
}
resampledPositions.AppendElement(resampledPosition);
}
if (anyPositionDifferentFromOriginal) {
// Store a copy of the original event, so that we can return to an
// non-resampled position later, if necessary.
mOriginalOfResampledTouchMove = Some(input);
// Add the original observed position to the historical data, as well as any
// leftover historical positions from the previous touch move event, and
// store the resampled values in the "final" position of the event.
PrependLeftoverHistoricalData(&input);
for (size_t i = 0; i < input.mTouches.Length(); i++) {
auto& touch = input.mTouches[i];
touch.mHistoricalData.AppendElement(SingleTouchData::HistoricalTouchData{
input.mTimeStamp,
touch.mScreenPoint,
touch.mLocalScreenPoint,
touch.mRadius,
touch.mRotationAngle,
touch.mForce,
});
// Remove any historical touch data that's in the future, compared to
// sampleTime. This data will be included by upcoming touch move
// events. This only happens if the frame timestamp can be older than the
// event timestamp, i.e. if interpolation occurs (rather than
// extrapolation).
auto futureDataStart = std::find_if(
touch.mHistoricalData.begin(), touch.mHistoricalData.end(),
[sampleTime](
const SingleTouchData::HistoricalTouchData& aHistoricalData) {
return aHistoricalData.mTimeStamp > sampleTime;
});
if (futureDataStart != touch.mHistoricalData.end()) {
nsTArray<SingleTouchData::HistoricalTouchData> futureData(
Span<SingleTouchData::HistoricalTouchData>(touch.mHistoricalData)
.From(futureDataStart.GetIndex()));
touch.mHistoricalData.TruncateLength(futureDataStart.GetIndex());
mRemainingTouchData.insert({touch.mIdentifier, std::move(futureData)});
}
touch.mScreenPoint = resampledPositions[i];
}
input.mTimeStamp = sampleTime;
}
EmitEvent(std::move(input), eventId);
mInResampledState = anyPositionDifferentFromOriginal;
}
void TouchResampler::PrependLeftoverHistoricalData(MultiTouchInput* aInput) {
for (auto& touch : aInput->mTouches) {
auto leftoverData = mRemainingTouchData.find(touch.mIdentifier);
if (leftoverData != mRemainingTouchData.end()) {
nsTArray<SingleTouchData::HistoricalTouchData> data =
std::move(leftoverData->second);
mRemainingTouchData.erase(leftoverData);
touch.mHistoricalData.InsertElementsAt(0, data);
}
if (TimeStamp cutoffTime = mLastEmittedEventTime) {
// If we received historical touch data that was further in the past than
// the last resampled event, discard that data so that the touch data
// points are emitted in order.
touch.mHistoricalData.RemoveElementsBy(
[cutoffTime](const SingleTouchData::HistoricalTouchData& aTouchData) {
return aTouchData.mTimeStamp < cutoffTime;
});
}
}
mRemainingTouchData.clear();
}
void TouchResampler::FlushDeferredTouchMoveEventsUnresampled() {
while (!mDeferredTouchMoveEvents.empty()) {
auto [input, eventId] = std::move(mDeferredTouchMoveEvents.front());
mDeferredTouchMoveEvents.pop();
PrependLeftoverHistoricalData(&input);
EmitEvent(std::move(input), eventId);
mInResampledState = false;
mOriginalOfResampledTouchMove = Nothing();
}
}
void TouchResampler::ReturnToNonResampledState() {
MOZ_RELEASE_ASSERT(mInResampledState);
MOZ_RELEASE_ASSERT(mDeferredTouchMoveEvents.empty(),
"Don't call this if there is a deferred touch move event. "
"We can return to the non-resampled state by sending that "
"event, rather than a copy of a previous event.");
// The last outgoing event was a resampled touch move event.
// Return to the non-resampled state, by sending a touch move event to
// "overwrite" any resampled positions with the original observed positions.
MultiTouchInput input = std::move(*mOriginalOfResampledTouchMove);
mOriginalOfResampledTouchMove = Nothing();
// For the event's timestamp, we want to backdate the correction as far as we
// can, while still preserving timestamp ordering. But we also don't want to
// backdate it to be older than it was originally.
if (mLastEmittedEventTime > input.mTimeStamp) {
input.mTimeStamp = mLastEmittedEventTime;
}
// Assemble the correct historical touch data for this event.
// We don't want to include data points that we've already sent out with the
// resampled event. And from the leftover data points, we only want those that
// don't duplicate the final time + position of this event.
for (auto& touch : input.mTouches) {
touch.mHistoricalData.Clear();
}
PrependLeftoverHistoricalData(&input);
for (auto& touch : input.mTouches) {
touch.mHistoricalData.RemoveElementsBy([&](const auto& histData) {
return histData.mTimeStamp >= input.mTimeStamp;
});
}
EmitExtraEvent(std::move(input));
mInResampledState = false;
}
void TouchResampler::TouchInfo::Update(const SingleTouchData& aTouch,
const TimeStamp& aEventTime) {
for (const auto& historicalData : aTouch.mHistoricalData) {
mBaseDataPoint = mLatestDataPoint;
mLatestDataPoint =
Some(DataPoint{historicalData.mTimeStamp, historicalData.mScreenPoint});
}
mBaseDataPoint = mLatestDataPoint;
mLatestDataPoint = Some(DataPoint{aEventTime, aTouch.mScreenPoint});
}
ScreenIntPoint TouchResampler::TouchInfo::ResampleAtTime(
const ScreenIntPoint& aLastObservedPosition, const TimeStamp& aTimeStamp) {
TimeStamp cutoff =
aTimeStamp - TimeDuration::FromMilliseconds(kTouchResampleWindowSize);
if (!mBaseDataPoint || !mLatestDataPoint ||
!(mBaseDataPoint->mTimeStamp < mLatestDataPoint->mTimeStamp) ||
mBaseDataPoint->mTimeStamp < cutoff) {
return aLastObservedPosition;
}
// For the actual resampling, connect the last two data points with a line and
// sample along that line.
TimeStamp t1 = mBaseDataPoint->mTimeStamp;
TimeStamp t2 = mLatestDataPoint->mTimeStamp;
double t = (aTimeStamp - t1) / (t2 - t1);
double x1 = mBaseDataPoint->mPosition.x;
double x2 = mLatestDataPoint->mPosition.x;
double y1 = mBaseDataPoint->mPosition.y;
double y2 = mLatestDataPoint->mPosition.y;
int32_t resampledX = round(x1 + t * (x2 - x1));
int32_t resampledY = round(y1 + t * (y2 - y1));
return ScreenIntPoint(resampledX, resampledY);
}
void TouchResampler::CurrentTouches::UpdateFromEvent(
const MultiTouchInput& aInput) {
switch (aInput.mType) {
case MultiTouchInput::MULTITOUCH_START: {
// A new touch has been added; make sure mTouches reflects the current
// touches in the event.
nsTArray<TouchInfo> newTouches;
for (const auto& touch : aInput.mTouches) {
const auto touchInfo = TouchByIdentifier(touch.mIdentifier);
if (touchInfo != mTouches.end()) {
// This is one of the existing touches.
newTouches.AppendElement(std::move(*touchInfo));
mTouches.RemoveElementAt(touchInfo);
} else {
// This is the new touch.
newTouches.AppendElement(TouchInfo{
touch.mIdentifier, Nothing(),
Some(DataPoint{aInput.mTimeStamp, touch.mScreenPoint})});
}
}
MOZ_ASSERT(mTouches.IsEmpty(), "Missing touch end before touch start?");
mTouches = std::move(newTouches);
break;
}
case MultiTouchInput::MULTITOUCH_MOVE: {
// The touches have moved.
// Add position information to the history data points.
for (const auto& touch : aInput.mTouches) {
const auto touchInfo = TouchByIdentifier(touch.mIdentifier);
MOZ_ASSERT(touchInfo != mTouches.end());
if (touchInfo != mTouches.end()) {
touchInfo->Update(touch, aInput.mTimeStamp);
}
}
mLatestDataPointTime = aInput.mTimeStamp;
break;
}
case MultiTouchInput::MULTITOUCH_END: {
// A touch has been removed.
MOZ_RELEASE_ASSERT(aInput.mTouches.Length() == 1);
const auto touchInfo = TouchByIdentifier(aInput.mTouches[0].mIdentifier);
MOZ_ASSERT(touchInfo != mTouches.end());
if (touchInfo != mTouches.end()) {
mTouches.RemoveElementAt(touchInfo);
}
break;
}
case MultiTouchInput::MULTITOUCH_CANCEL:
// All touches are canceled.
mTouches.Clear();
break;
}
}
nsTArray<TouchResampler::TouchInfo>::iterator
TouchResampler::CurrentTouches::TouchByIdentifier(int32_t aIdentifier) {
return std::find_if(mTouches.begin(), mTouches.end(),
[aIdentifier](const TouchInfo& info) {
return info.mIdentifier == aIdentifier;
});
}
ScreenIntPoint TouchResampler::CurrentTouches::ResampleTouchPositionAtTime(
int32_t aIdentifier, const ScreenIntPoint& aLastObservedPosition,
const TimeStamp& aTimeStamp) {
const auto touchInfo = TouchByIdentifier(aIdentifier);
MOZ_ASSERT(touchInfo != mTouches.end());
if (touchInfo != mTouches.end()) {
return touchInfo->ResampleAtTime(aLastObservedPosition, aTimeStamp);
}
return aLastObservedPosition;
}
} // namespace widget
} // namespace mozilla