Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

terminal: support mouse input #13711

Merged
merged 8 commits into from
Apr 17, 2024
Merged

terminal: support mouse input #13711

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
fe0cb03
terminal-unix: support mouse escape codes
na-na-hi Mar 2, 2024
a2da10b
terminal-unix: ignore unhandled mouse CSI sequences
na-na-hi Mar 2, 2024
39544cf
vo_{tct,sixel,kitty}: implement mouse support
na-na-hi Mar 2, 2024
63ae764
terminal-unix: style fix
na-na-hi Mar 14, 2024
b8c2bdd
terminal: add terminal_set_mouse_input function
na-na-hi Mar 31, 2024
b106cb5
terminal-win: implement terminal_set_mouse_input
na-na-hi Mar 31, 2024
62c6258
terminal-win: implement terminal_get_size2
na-na-hi Mar 31, 2024
96a805a
terminal-win: support mouse input events
na-na-hi Mar 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 55 additions & 4 deletions osdep/terminal-unix.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,13 +54,23 @@ static struct termios tio_orig;

static int tty_in = -1, tty_out = -1;

enum entry_type {
ENTRY_TYPE_KEY = 0,
ENTRY_TYPE_MOUSE_BUTTON,
ENTRY_TYPE_MOUSE_MOVE,
};

struct key_entry {
const char *seq;
int mpkey;
// If this is not NULL, then if seq is matched as unique prefix, the
// existing sequence is replaced by the following string. Matching
// continues normally, and mpkey is or-ed into the final result.
const char *replace;
// Extend the match length by a certain length, so the contents
// after the match can be processed with custom logic.
int skip;
enum entry_type type;
};

static const struct key_entry keys[] = {
Expand Down Expand Up @@ -160,6 +170,18 @@ static const struct key_entry keys[] = {
{"\033[29~", MP_KEY_MENU},
{"\033[Z", MP_KEY_TAB | MP_KEY_MODIFIER_SHIFT},

// Mouse button inputs. 2 bytes of position information requires special processing.
{"\033[M ", MP_MBTN_LEFT | MP_KEY_STATE_DOWN, .skip = 2, .type = ENTRY_TYPE_MOUSE_BUTTON},
{"\033[M!", MP_MBTN_MID | MP_KEY_STATE_DOWN, .skip = 2, .type = ENTRY_TYPE_MOUSE_BUTTON},
{"\033[M\"", MP_MBTN_RIGHT | MP_KEY_STATE_DOWN, .skip = 2, .type = ENTRY_TYPE_MOUSE_BUTTON},
{"\033[M#", MP_INPUT_RELEASE_ALL, .skip = 2, .type = ENTRY_TYPE_MOUSE_BUTTON},
{"\033[M`", MP_WHEEL_UP, .skip = 2, .type = ENTRY_TYPE_MOUSE_BUTTON},
{"\033[Ma", MP_WHEEL_DOWN, .skip = 2, .type = ENTRY_TYPE_MOUSE_BUTTON},
// Mouse move inputs. No key events should be generated for them.
{"\033[M@", MP_MBTN_LEFT | MP_KEY_STATE_DOWN, .skip = 2, .type = ENTRY_TYPE_MOUSE_MOVE},
{"\033[MA", MP_MBTN_MID | MP_KEY_STATE_DOWN, .skip = 2, .type = ENTRY_TYPE_MOUSE_MOVE},
{"\033[MB", MP_MBTN_RIGHT | MP_KEY_STATE_DOWN, .skip = 2, .type = ENTRY_TYPE_MOUSE_MOVE},
{"\033[MC", MP_INPUT_RELEASE_ALL, .skip = 2, .type = ENTRY_TYPE_MOUSE_MOVE},
{0}
};

Expand Down Expand Up @@ -218,6 +240,13 @@ static void process_input(struct input_ctx *input_ctx, bool timeout)
int mods = 0;
if (buf.b[0] == '\033') {
if (buf.len > 1 && buf.b[1] == '[') {
// Throw away unrecognized mouse CSI sequences.
// Cannot be handled by the loop below since the bytes
// afterwards can be out of that range.
if (buf.len > 2 && buf.b[2] == 'M') {
skip_buf(&buf, buf.len);
continue;
}
// unknown CSI sequence. wait till it completes
for (int i = 2; i < buf.len; i++) {
if (buf.b[i] >= 0x40 && buf.b[i] <= 0x7E) {
Expand Down Expand Up @@ -250,7 +279,7 @@ static void process_input(struct input_ctx *input_ctx, bool timeout)
continue;
}

int seq_len = strlen(match->seq);
int seq_len = strlen(match->seq) + match->skip;
if (seq_len > buf.len)
goto read_more; /* partial match */

Expand All @@ -264,7 +293,23 @@ static void process_input(struct input_ctx *input_ctx, bool timeout)
continue;
}

mp_input_put_key(input_ctx, buf.mods | match->mpkey);
// Parse the initially skipped mouse position information.
// The positions are 1-based character cell positions plus 32.
// Treat mouse position as the pixel values at the center of the cell.
if ((match->type == ENTRY_TYPE_MOUSE_BUTTON ||
match->type == ENTRY_TYPE_MOUSE_MOVE) && seq_len >= 6)
{
int num_rows = 80;
int num_cols = 25;
int total_px_width = 0;
int total_px_height = 0;
terminal_get_size2(&num_rows, &num_cols, &total_px_width, &total_px_height);
mp_input_set_mouse_pos(input_ctx,
(buf.b[4] - 32.5) * (total_px_width / num_cols),
(buf.b[5] - 32.5) * (total_px_height / num_rows));
}
if (match->type != ENTRY_TYPE_MOUSE_MOVE)
mp_input_put_key(input_ctx, buf.mods | match->mpkey);
skip_buf(&buf, seq_len);
}

Expand Down Expand Up @@ -296,12 +341,12 @@ static void do_activate_getch2(void)
enable_kx(true);

struct termios tio_new;
tcgetattr(tty_in,&tio_new);
tcgetattr(tty_in, &tio_new);

tio_new.c_lflag &= ~(ICANON|ECHO); /* Clear ICANON and ECHO. */
tio_new.c_cc[VMIN] = 1;
tio_new.c_cc[VTIME] = 0;
tcsetattr(tty_in,TCSANOW,&tio_new);
tcsetattr(tty_in, TCSANOW, &tio_new);

getch2_active = 1;
}
Expand Down Expand Up @@ -543,6 +588,12 @@ void terminal_get_size2(int *rows, int *cols, int *px_width, int *px_height)
*px_height = ws.ws_ypixel;
}

void terminal_set_mouse_input(bool enable)
{
printf(enable ? TERM_ESC_ENABLE_MOUSE : TERM_ESC_DISABLE_MOUSE);
fflush(stdout);
}

void terminal_init(void)
{
assert(!getch2_enabled);
Expand Down
136 changes: 106 additions & 30 deletions osdep/terminal-win.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,15 +111,35 @@ static bool is_native_out_vt(HANDLE hOut)
void terminal_get_size(int *w, int *h)
{
CONSOLE_SCREEN_BUFFER_INFO cinfo;
HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE);
HANDLE hOut = hSTDOUT;
if (GetConsoleScreenBufferInfo(hOut, &cinfo)) {
*w = cinfo.dwMaximumWindowSize.X - (is_native_out_vt(hOut) ? 0 : 1);
*h = cinfo.dwMaximumWindowSize.Y;
}
}

static bool get_font_size(int *w, int *h)
{
CONSOLE_FONT_INFO finfo;
HANDLE hOut = hSTDOUT;
BOOL res = GetCurrentConsoleFont(hOut, FALSE, &finfo);
if (res) {
*w = finfo.dwFontSize.X;
*h = finfo.dwFontSize.Y;
}
return res;
}

void terminal_get_size2(int *rows, int *cols, int *px_width, int *px_height)
{
int w = 0, h = 0, fw = 0, fh = 0;
terminal_get_size(&w, &h);
if (get_font_size(&fw, &fh)) {
*px_width = fw * w;
*px_height = fh * h;
*rows = w;
*cols = h;
}
Comment on lines +135 to +142
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@na-na-hi

This doesn't work well for recently added sixel support in conhost.

They use 10x20 sized cells (see microsoft/terminal#17504).

Do you want to play around with it and improve it?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I looked into it and the problem is with the current terminal input architecture. mpv never used control sequences for querying, and all terminal inputs are processed in a dedicated thread. That means with the current architecture, querying character pixel size would be an asynchronous operation since there might be some inputs waiting to be processed before the query responses arrive.

Not sure what's the best way to proceed from here.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure what's the best way to proceed from here.

Same. Would be nice to have it supported, but don't see easy path forward with this one.

}

static bool has_input_events(HANDLE h)
Expand All @@ -139,35 +159,80 @@ static void read_input(HANDLE in)
break;

// Only key-down events are interesting to us
if (event.EventType != KEY_EVENT)
continue;
KEY_EVENT_RECORD *record = &event.Event.KeyEvent;
if (!record->bKeyDown)
continue;

UINT vkey = record->wVirtualKeyCode;
bool ext = record->dwControlKeyState & ENHANCED_KEY;

int mods = 0;
if (record->dwControlKeyState & (LEFT_ALT_PRESSED | RIGHT_ALT_PRESSED))
mods |= MP_KEY_MODIFIER_ALT;
if (record->dwControlKeyState & (LEFT_CTRL_PRESSED | RIGHT_CTRL_PRESSED))
mods |= MP_KEY_MODIFIER_CTRL;
if (record->dwControlKeyState & SHIFT_PRESSED)
mods |= MP_KEY_MODIFIER_SHIFT;

int mpkey = mp_w32_vkey_to_mpkey(vkey, ext);
if (mpkey) {
mp_input_put_key(input_ctx, mpkey | mods);
} else {
// Only characters should be remaining
int c = record->uChar.UnicodeChar;
// The ctrl key always produces control characters in the console.
// Shift them back up to regular characters.
if (c > 0 && c < 0x20 && (mods & MP_KEY_MODIFIER_CTRL))
c += (mods & MP_KEY_MODIFIER_SHIFT) ? 0x40 : 0x60;
if (c >= 0x20)
mp_input_put_key(input_ctx, c | mods);
switch (event.EventType)
{
case KEY_EVENT: {
KEY_EVENT_RECORD *record = &event.Event.KeyEvent;
if (!record->bKeyDown)
continue;

UINT vkey = record->wVirtualKeyCode;
bool ext = record->dwControlKeyState & ENHANCED_KEY;

int mods = 0;
if (record->dwControlKeyState & (LEFT_ALT_PRESSED | RIGHT_ALT_PRESSED))
mods |= MP_KEY_MODIFIER_ALT;
if (record->dwControlKeyState & (LEFT_CTRL_PRESSED | RIGHT_CTRL_PRESSED))
mods |= MP_KEY_MODIFIER_CTRL;
if (record->dwControlKeyState & SHIFT_PRESSED)
mods |= MP_KEY_MODIFIER_SHIFT;

int mpkey = mp_w32_vkey_to_mpkey(vkey, ext);
if (mpkey) {
mp_input_put_key(input_ctx, mpkey | mods);
} else {
// Only characters should be remaining
int c = record->uChar.UnicodeChar;
// The ctrl key always produces control characters in the console.
// Shift them back up to regular characters.
if (c > 0 && c < 0x20 && (mods & MP_KEY_MODIFIER_CTRL))
c += (mods & MP_KEY_MODIFIER_SHIFT) ? 0x40 : 0x60;
if (c >= 0x20)
mp_input_put_key(input_ctx, c | mods);
}
break;
}
case MOUSE_EVENT: {
MOUSE_EVENT_RECORD *record = &event.Event.MouseEvent;
int mods = 0;
if (record->dwControlKeyState & (LEFT_ALT_PRESSED | RIGHT_ALT_PRESSED))
mods |= MP_KEY_MODIFIER_ALT;
if (record->dwControlKeyState & (LEFT_CTRL_PRESSED | RIGHT_CTRL_PRESSED))
mods |= MP_KEY_MODIFIER_CTRL;
if (record->dwControlKeyState & SHIFT_PRESSED)
mods |= MP_KEY_MODIFIER_SHIFT;

switch (record->dwEventFlags) {
case MOUSE_MOVED: {
int w = 0, h = 0;
if (get_font_size(&w, &h)) {
mp_input_set_mouse_pos(input_ctx, w * (record->dwMousePosition.X + 0.5),
h * (record->dwMousePosition.Y + 0.5));
}
break;
}
case MOUSE_HWHEELED: {
int button = (int16_t)HIWORD(record->dwButtonState) > 0 ? MP_WHEEL_RIGHT : MP_WHEEL_LEFT;
mp_input_put_key(input_ctx, button | mods);
break;
}
case MOUSE_WHEELED: {
int button = (int16_t)HIWORD(record->dwButtonState) > 0 ? MP_WHEEL_UP : MP_WHEEL_DOWN;
mp_input_put_key(input_ctx, button | mods);
break;
}
default: {
int left_button_state = record->dwButtonState & FROM_LEFT_1ST_BUTTON_PRESSED ?
MP_KEY_STATE_DOWN : MP_KEY_STATE_UP;
mp_input_put_key(input_ctx, MP_MBTN_LEFT | mods | left_button_state);
int right_button_state = record->dwButtonState & RIGHTMOST_BUTTON_PRESSED ?
MP_KEY_STATE_DOWN : MP_KEY_STATE_UP;
mp_input_put_key(input_ctx, MP_MBTN_RIGHT | mods | right_button_state);
break;
}
}
break;
}
}
}
}
Expand Down Expand Up @@ -556,6 +621,17 @@ bool terminal_try_attach(void)
return true;
}

void terminal_set_mouse_input(bool enable)
{
DWORD cmode;
HANDLE in = hSTDIN;
if (GetConsoleMode(in, &cmode)) {
cmode = enable ? cmode | ENABLE_MOUSE_INPUT
: cmode & (~ENABLE_MOUSE_INPUT);
SetConsoleMode(in, cmode);
}
}

void terminal_init(void)
{
CONSOLE_SCREEN_BUFFER_INFO cinfo;
Expand Down
6 changes: 6 additions & 0 deletions osdep/terminal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@
#define TERM_ESC_ALT_SCREEN "\033[?1049h"
#define TERM_ESC_NORMAL_SCREEN "\033[?1049l"

#define TERM_ESC_ENABLE_MOUSE "\033[?1003h"
#define TERM_ESC_DISABLE_MOUSE "\033[?1003l"

struct input_ctx;

/* Global initialization for terminal output. */
Expand All @@ -55,6 +58,9 @@ void terminal_get_size(int *w, int *h);
/* Get terminal-size in columns/rows and width/height in pixels. */
void terminal_get_size2(int *rows, int *cols, int *px_width, int *px_height);

/* Enable/Disable mouse input. */
void terminal_set_mouse_input(bool enable);

// Windows only.
int mp_console_vfprintf(void *wstream, const char *format, va_list args);
int mp_console_write(void *wstream, bstr str);
Expand Down
2 changes: 2 additions & 0 deletions video/out/vo_kitty.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,7 @@ static int preinit(struct vo *vo)
#endif

write_str(TERM_ESC_HIDE_CURSOR);
terminal_set_mouse_input(true);
if (p->opts.alt_screen)
write_str(TERM_ESC_ALT_SCREEN);

Expand Down Expand Up @@ -389,6 +390,7 @@ static void uninit(struct vo *vo)
#endif

write_str(TERM_ESC_RESTORE_CURSOR);
terminal_set_mouse_input(false);

if (p->opts.alt_screen) {
write_str(TERM_ESC_NORMAL_SCREEN);
Expand Down
2 changes: 2 additions & 0 deletions video/out/vo_sixel.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,7 @@ static int preinit(struct vo *vo)
sixel_strwrite(TERM_ESC_ALT_SCREEN);

sixel_strwrite(TERM_ESC_HIDE_CURSOR);
terminal_set_mouse_input(true);

/* don't use private color registers for each frame. */
sixel_strwrite(TERM_ESC_USE_GLOBAL_COLOR_REG);
Expand Down Expand Up @@ -559,6 +560,7 @@ static void uninit(struct vo *vo)
struct priv *priv = vo->priv;

sixel_strwrite(TERM_ESC_RESTORE_CURSOR);
terminal_set_mouse_input(false);

if (priv->opts.alt_screen)
sixel_strwrite(TERM_ESC_NORMAL_SCREEN);
Expand Down
2 changes: 2 additions & 0 deletions video/out/vo_tct.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ static void flip_page(struct vo *vo)
static void uninit(struct vo *vo)
{
WRITE_STR(TERM_ESC_RESTORE_CURSOR);
terminal_set_mouse_input(false);
WRITE_STR(TERM_ESC_NORMAL_SCREEN);
struct priv *p = vo->priv;
talloc_free(p->frame);
Expand Down Expand Up @@ -328,6 +329,7 @@ static int preinit(struct vo *vo)
}

WRITE_STR(TERM_ESC_HIDE_CURSOR);
terminal_set_mouse_input(true);
WRITE_STR(TERM_ESC_ALT_SCREEN);

return 0;
Expand Down