Skip to content

feat(ui): Enhances virtual keyboard with sticky modifier key support #500

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

feat(ui): Enhances virtual keyboard with sticky modifier key support #500

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions ui/src/components/InfoBar.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ export default function InfoBar() {
const keyboardLedState = useHidStore(state => state.keyboardLedState);
const keyboardLedStateSyncAvailable = useHidStore(state => state.keyboardLedStateSyncAvailable);
const keyboardLedSync = useSettingsStore(state => state.keyboardLedSync);

const isShiftActive = useHidStore(state => state.isShiftActive);
const isCtrlActive = useHidStore(state => state.isCtrlActive);
const isAltActive = useHidStore(state => state.isAltActive);
const isMetaActive = useHidStore(state => state.isMetaActive);
const isAltGrActive = useHidStore(state => state.isAltGrActive);

const isTurnServerInUse = useRTCStore(state => state.isTurnServerInUse);

Expand Down Expand Up @@ -135,6 +141,56 @@ export default function InfoBar() {
{keyboardLedSync === "browser" ? "Browser" : "Host"}
</div>
) : null}
<div
className={cx(
"shrink-0 p-1 px-1.5 text-xs",
isShiftActive
? "text-black dark:text-white"
: "text-slate-800/20 dark:text-slate-300/20",
)}
>
Shift
</div>
<div
className={cx(
"shrink-0 p-1 px-1.5 text-xs",
isCtrlActive
? "text-black dark:text-white"
: "text-slate-800/20 dark:text-slate-300/20",
)}
>
Ctrl
</div>
<div
className={cx(
"shrink-0 p-1 px-1.5 text-xs",
isAltActive
? "text-black dark:text-white"
: "text-slate-800/20 dark:text-slate-300/20",
)}
>
Alt
</div>
<div
className={cx(
"shrink-0 p-1 px-1.5 text-xs",
isMetaActive
? "text-black dark:text-white"
: "text-slate-800/20 dark:text-slate-300/20",
)}
>
Meta
</div>
<div
className={cx(
"shrink-0 p-1 px-1.5 text-xs",
isAltGrActive
? "text-black dark:text-white"
: "text-slate-800/20 dark:text-slate-300/20",
)}
>
AltGr
</div>
<div
className={cx(
"shrink-0 p-1 px-1.5 text-xs",
Expand Down
206 changes: 133 additions & 73 deletions ui/src/components/VirtualKeyboard.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ const AttachIcon = ({ className }: { className?: string }) => {

function KeyboardWrapper() {
const [layoutName, setLayoutName] = useState("default");
const [depressedButtons, setDepressedButtons] = useState("");

const keyboardRef = useRef<HTMLDivElement>(null);
const showAttachedVirtualKeyboard = useUiStore(
Expand Down Expand Up @@ -54,6 +55,21 @@ function KeyboardWrapper() {

const setIsCapsLockActive = useHidStore(state => state.setIsCapsLockActive);

const isShiftActive = useHidStore(state => state.isShiftActive);
const setIsShiftActive = useHidStore(state => state.setIsShiftActive);

const isCtrlActive = useHidStore(state => state.isCtrlActive);
const setIsCtrlActive = useHidStore(state => state.setIsCtrlActive);

const isAltActive = useHidStore(state => state.isAltActive);
const setIsAltActive = useHidStore(state => state.setIsAltActive);

const isMetaActive = useHidStore(state => state.isMetaActive);
const setIsMetaActive = useHidStore(state => state.setIsMetaActive);

const isAltGrActive = useHidStore(state => state.isAltGrActive);
const setIsAltGrActive = useHidStore(state => state.setIsAltGrActive);

const startDrag = useCallback((e: MouseEvent | TouchEvent) => {
if (!keyboardRef.current) return;
if (e instanceof TouchEvent && e.touches.length > 1) return;
Expand Down Expand Up @@ -123,80 +139,123 @@ function KeyboardWrapper() {
};
}, [endDrag, onDrag, startDrag]);

const onKeyDown = useCallback(
(key: string) => {
const isKeyShift = key === "{shift}" || key === "ShiftLeft" || key === "ShiftRight";
const isKeyCaps = key === "CapsLock";
const cleanKey = key.replace(/[()]/g, "");
const keyHasShiftModifier = key.includes("(");

// Handle toggle of layout for shift or caps lock
const toggleLayout = () => {
setLayoutName(prevLayout => (prevLayout === "default" ? "shift" : "default"));
};

if (key === "CtrlAltDelete") {
sendKeyboardEvent(
[keys["Delete"]],
[modifiers["ControlLeft"], modifiers["AltLeft"]],
);
setTimeout(resetKeyboardState, 100);
return;
}
useEffect(() => {
// if you have the CapsLock "down", then the shift state is inverted
const effectiveShift = isCapsLockActive ? false === isShiftActive : isShiftActive;
setLayoutName(effectiveShift ? "shift" : "default");
},
[setLayoutName, isCapsLockActive, isShiftActive]
);

if (key === "AltMetaEscape") {
sendKeyboardEvent(
[keys["Escape"]],
[modifiers["MetaLeft"], modifiers["AltLeft"]],
);
// this causes the buttons to look depressed/clicked depending on the sticky state
useEffect(() => {
let buttons = "None "; // make sure we name at least one (fake) button
if (isCapsLockActive) buttons += "CapsLock ";
if (isShiftActive) buttons += "ShiftLeft ShiftRight ";
if (isCtrlActive) buttons += "ControlLeft ControlRight ";
if (isAltActive) buttons += "AltLeft AltRight ";
if (isMetaActive) buttons += "MetaLeft MetaRight ";
setDepressedButtons(buttons.trimEnd());
},
[setDepressedButtons, isCapsLockActive, isShiftActive, isCtrlActive, isAltActive, isMetaActive, isAltGrActive]
);

setTimeout(resetKeyboardState, 100);
return;
}
const onKeyPress = useCallback((key: string) => {
// handle the fake combo keys first
if (key === "CtrlAltDelete") {
sendKeyboardEvent(
[keys["Delete"]],
[modifiers["ControlLeft"], modifiers["AltLeft"]],
);
setTimeout(resetKeyboardState, 100);
return;
}

if (key === "CtrlAltBackspace") {
sendKeyboardEvent(
[keys["Backspace"]],
[modifiers["ControlLeft"], modifiers["AltLeft"]],
);
if (key === "AltMetaEscape") {
sendKeyboardEvent(
[keys["Escape"]],
[modifiers["MetaLeft"], modifiers["AltLeft"]],
);

setTimeout(resetKeyboardState, 100);
return;
}
setTimeout(resetKeyboardState, 100);
return;
}

if (isKeyShift || isKeyCaps) {
toggleLayout();
if (key === "CtrlAltBackspace") {
sendKeyboardEvent(
[keys["Backspace"]],
[modifiers["ControlLeft"], modifiers["AltLeft"]],
);

if (isCapsLockActive) {
if (!isKeyboardLedManagedByHost) {
setIsCapsLockActive(false);
}
sendKeyboardEvent([keys["CapsLock"]], []);
return;
}
}
setTimeout(resetKeyboardState, 100);
return;
}

// Handle caps lock state change
if (isKeyCaps && !isKeyboardLedManagedByHost) {
setIsCapsLockActive(!isCapsLockActive);
}
// strip away the parens for shifted characters
const cleanKey = key.replace(/[()]/g, "");

const passthrough = ["PrintScreen", "SystemRequest", "Pause", "Break", "ScrollLock", "Enter", "Space"].find((value) => value === cleanKey);

if (passthrough) {
emitkeycode(cleanKey);
return;
}

// adjust the sticky state of the Shift/Ctrl/Alt/Meta/AltGr
if (key === "CapsLock" && !isKeyboardLedManagedByHost)
setIsCapsLockActive(!isCapsLockActive);
else if (key === "ShiftLeft" || key === "ShiftRight")
setIsShiftActive(!isShiftActive);
else if (key === "ControlLeft" || key === "ControlRight")
setIsCtrlActive(!isCtrlActive);
else if (key === "AltLeft" || key === "AltRight")
setIsAltActive(!isAltActive);
else if (key === "MetaLeft" || key === "MetaRight")
setIsMetaActive(!isMetaActive);
else if (key === "AltGr")
setIsAltGrActive(!isAltGrActive);

emitkeycode(cleanKey);

function emitkeycode(key: string) {
const effectiveMods: number[] = [];

if (isShiftActive)
effectiveMods.push(modifiers["ShiftLeft"]);

// Collect new active keys and modifiers
const newKeys = keys[cleanKey] ? [keys[cleanKey]] : [];
const newModifiers =
keyHasShiftModifier && !isCapsLockActive ? [modifiers["ShiftLeft"]] : [];
if (isCtrlActive)
effectiveMods.push(modifiers["ControlLeft"]);

// Update current keys and modifiers
sendKeyboardEvent(newKeys, newModifiers);
if (isAltActive)
effectiveMods.push(modifiers["AltLeft"]);

// If shift was used as a modifier and caps lock is not active, revert to default layout
if (keyHasShiftModifier && !isCapsLockActive) {
setLayoutName("default");
if (isMetaActive)
effectiveMods.push(modifiers["MetaLeft"]);

if (isAltGrActive) {
effectiveMods.push(modifiers["MetaRight"]);
effectiveMods.push(modifiers["CtrlLeft"]);
}

setTimeout(resetKeyboardState, 100);
},
[isCapsLockActive, isKeyboardLedManagedByHost, sendKeyboardEvent, resetKeyboardState, setIsCapsLockActive],
const keycode = keys[key];
if (keycode) {
// send the keycode with modifiers
sendKeyboardEvent([keycode], effectiveMods);
}

// release the key (if one pressed), but retain the modifiers
setTimeout(() => sendKeyboardEvent([], effectiveMods), 50);
}
},
[isKeyboardLedManagedByHost,
setIsCapsLockActive, isCapsLockActive,
setIsShiftActive, isShiftActive,
setIsCtrlActive, isCtrlActive,
setIsAltActive, isAltActive,
setIsMetaActive, isMetaActive,
setIsAltGrActive, isAltGrActive,
sendKeyboardEvent, resetKeyboardState
],
);

const virtualKeyboard = useHidStore(state => state.isVirtualKeyboardEnabled);
Expand Down Expand Up @@ -276,12 +335,16 @@ function KeyboardWrapper() {
<Keyboard
baseClass="simple-keyboard-main"
layoutName={layoutName}
onKeyPress={onKeyDown}
onKeyPress={onKeyPress}
buttonTheme={[
{
class: "combination-key",
buttons: "CtrlAltDelete AltMetaEscape CtrlAltBackspace",
},
{
class: "depressed-key",
buttons: depressedButtons
},
]}
display={keyDisplayMap}
layout={{
Expand All @@ -305,34 +368,31 @@ function KeyboardWrapper() {
],
}}
disableButtonHold={true}
syncInstanceInputs={true}
debug={false}
/>

<div className="controlArrows">
<Keyboard
baseClass="simple-keyboard-control"
theme="simple-keyboard hg-theme-default hg-layout-default"
layoutName={layoutName}
onKeyPress={onKeyDown}
onKeyPress={onKeyPress}
display={keyDisplayMap}
layout={{
default: ["PrintScreen ScrollLock Pause", "Insert Home Pageup", "Delete End Pagedown"],
shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home Pageup", "Delete End Pagedown"],
default: ["PrintScreen ScrollLock Pause", "Insert Home PageUp", "Delete End PageDown"],
shift: ["(PrintScreen) ScrollLock (Pause)", "Insert Home PageUp", "Delete End PageDown"],
}}
syncInstanceInputs={true}
debug={false}
disableButtonHold={true}
/>
<Keyboard
baseClass="simple-keyboard-arrows"
theme="simple-keyboard hg-theme-default hg-layout-default"
onKeyPress={onKeyDown}
onKeyPress={onKeyPress}
display={keyDisplayMap}
layout={{
default: ["ArrowUp", "ArrowLeft ArrowDown ArrowRight"],
shift: ["ArrowUp", "ArrowLeft ArrowDown ArrowRight"],
}}
syncInstanceInputs={true}
debug={false}
disableButtonHold={true}
/>
</div>
</div>
Expand Down
Loading