Skip to content

fix: inject real MotionEvents for drag and scroll #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sammyjeng wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: inject real MotionEvents for drag and scroll #7

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 65 additions & 4 deletions app/src/main/java/com/appknox/vnc/InputService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
public class InputService extends AccessibilityService {
static InputWrapperService inputWrapperService;

// Minimum pointer travel (px) to treat a gesture as a drag/scroll rather than a tap.
private static final int TOUCH_SLOP = 10;

/**
* This tracks gesture completion per client.
*/
Expand All @@ -48,6 +51,8 @@ static class InputContext {
boolean isButtonOneDown;
Path path = new Path();
long lastGestureStartTime;
int gestureStartX;
int gestureStartY;
GestureCallback gestureCallback = new GestureCallback();
InputPointerView pointerView;
// keyboard-related
Expand Down Expand Up @@ -337,6 +342,8 @@ public static void onCutText(String text, long client) {
private void startGesture(InputContext inputContext, int x, int y) {
inputContext.path.reset();
inputContext.path.moveTo( x, y );
inputContext.gestureStartX = x;
inputContext.gestureStartY = y;
inputContext.lastGestureStartTime = System.currentTimeMillis();
}

Expand All @@ -345,18 +352,31 @@ private void continueGesture(InputContext inputContext, int x, int y) {
}

private void endGesture(InputContext inputContext, int x, int y) {
inputContext.path.lineTo( x, y );
long duration = System.currentTimeMillis() - inputContext.lastGestureStartTime;
// gesture ended very very shortly after start (< 1ms). make it 1ms to get dispatched to the system
if (duration == 0) duration = 1;

int dx = x - inputContext.gestureStartX;
int dy = y - inputContext.gestureStartY;

// Real drag/scroll: inject a genuine MotionEvent swipe via root. Apps that
// render their own scrollables (e.g. Flutter) ignore dispatchGesture()'s
// synthetic strokes but react to real touch events, exactly like scrcpy.
if (Math.abs(dx) > TOUCH_SLOP || Math.abs(dy) > TOUCH_SLOP) {
long swipeMs = Math.max(50, Math.min(duration, 3000));
if (execInput("swipe " + inputContext.gestureStartX + " " + inputContext.gestureStartY
+ " " + x + " " + y + " " + swipeMs))
return;
}

// Tap, or root/input unavailable: fall back to the synthetic gesture,
// which works fine for taps.
inputContext.path.lineTo( x, y );
GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription( inputContext.path, 0, duration);
GestureDescription.Builder builder = new GestureDescription.Builder();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
builder.setDisplayId(inputContext.getDisplayId());
}
builder.addStroke(stroke);
// Docs says: Any gestures currently in progress, whether from the user, this service, or another service, will be cancelled.
// But at least on API level 32, setting different display ids with the builder allows for parallel input.
dispatchGesture(builder.build(), null, null);
}

Expand All @@ -368,7 +388,17 @@ private void longPress(InputContext inputContext, int x, int y )

private void scroll(InputContext inputContext, int x, int y, int scrollAmount )
{
// Inject a real MotionEvent swipe via root. Apps rendering their own
// scrollables (e.g. Flutter) ignore dispatchGesture() swipes but scroll
// on real touch. scrollAmount > 0 -> content down, < 0 -> up.
int h = getRealHeight();
int y1 = clamp(y, 1, h - 1);
int y2 = clamp(y - scrollAmount, 1, h - 1);
if (execInput("swipe " + x + " " + y1 + " " + x + " " + y2 + " 250"))
return;

/*
Fallback: synthetic swipe gesture.
Ignore if another gesture is still ongoing. Especially true for scroll events:
These mouse button 4,5 events come per each virtual scroll wheel click, an incoming
event would cancel the preceding one, only actually scrolling when the user stopped
Expand All @@ -381,6 +411,37 @@ private void scroll(InputContext inputContext, int x, int y, int scrollAmount )
dispatchGesture(createSwipe(inputContext, x, y, x, y - scrollAmount, ViewConfiguration.getScrollDefaultDelay()), inputContext.gestureCallback, null);
}

/**
* Run an `input` sub-command as root, the same mechanism the app already uses
* for key events, so the injected touch events are real MotionEvents that
* every app (including Flutter) reacts to.
*/
private static boolean execInput(String args)
{
try {
Process p = Runtime.getRuntime().exec(new String[]{ "su", "-c", "input " + args });
int rc = p.waitFor();
Log.d(TAG, "execInput 'input " + args + "' rc=" + rc);
return rc == 0;
} catch (Exception e) {
Log.e(TAG, "execInput failed for 'input " + args + "': " + e);
return false;
}
}

private static int clamp(int v, int lo, int hi)
{
return Math.max(lo, Math.min(v, hi));
}

private int getRealHeight()
{
DisplayMetrics dm = new DisplayMetrics();
WindowManager wm = (WindowManager) getApplicationContext().getSystemService(Context.WINDOW_SERVICE);
wm.getDefaultDisplay().getRealMetrics(dm);
return dm.heightPixels;
}

private static GestureDescription createClick(InputContext inputContext, int x, int y, int duration )
{
Path clickPath = new Path();
Expand Down
Loading