working on the embeddable clapper player

package.json

@@ -43,6 +43,5 @@
     "packages/app"
   ],
   "dependencies": {
-    "civitai": "^0.1.15"
   }
 }

packages/app/package.json

@@ -89,6 +89,8 @@
     "@xyflow/react": "^12.0.3",
     "autoprefixer": "10.4.19",
     "base64-arraybuffer": "^1.0.2",
+    "bellhop-iframe": "^3.5.0",
+    "civitai": "^0.1.15",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
     "cmdk": "^0.2.1",

packages/app/src/app/api/resolve/providers/comfyui/index.ts

@@ -1,5 +1,9 @@
 import { ResolveRequest } from '@aitube/clapper-services'
-import { ClapAssetSource, ClapSegmentCategory, generateSeed } from '@aitube/clap'
+import {
+  ClapAssetSource,
+  ClapSegmentCategory,
+  generateSeed,
+} from '@aitube/clap'
 import { TimelineSegment } from '@aitube/timeline'
 import { BasicCredentials, CallWrapper, ComfyApi } from '@saintno/comfyui-sdk'
 import { decodeOutput } from '@/lib/utils/decodeOutput'

packages/app/src/app/embed/EmbeddedPlayer.tsx

@@ -0,0 +1,110 @@
+import { useEffect, useRef, useState } from 'react'
+
+import { cn } from '@/lib/utils'
+
+import { TogglePlayback } from './TogglePlayback'
+import { StaticOrInteractiveTag } from './StaticOrInteractive'
+import { useMonitor } from '@/services'
+import { useTimeline } from '@aitube/timeline'
+
+export function EmbeddedPlayer() {
+  const isPlaying = useMonitor((s) => s.isPlaying)
+  const togglePlayback = useMonitor((s) => s.togglePlayback)
+
+  const meta = useTimeline((s) => s.meta)
+
+  const [isOverlayVisible, setOverlayVisible] = useState(true)
+
+  const overlayTimerRef = useRef<NodeJS.Timeout>()
+  // const videoLayerRef = useRef<HTMLDivElement>(null)
+  // const segmentationLayerRef = useRef<HTMLDivElement>(null)
+
+  const isPlayingRef = useRef(isPlaying)
+  isPlayingRef.current = isPlaying
+
+  const scheduleOverlayInvisibility = () => {
+    clearTimeout(overlayTimerRef.current)
+    overlayTimerRef.current = setTimeout(() => {
+      if (isPlayingRef.current) {
+        setOverlayVisible(!isPlayingRef.current)
+      }
+      clearTimeout(overlayTimerRef.current)
+    }, 3000)
+  }
+
+  return (
+    <>
+      {/* content overlay, with the gradient, buttons etc */}
+      <div
+        className={cn(
+          `pointer-events-none absolute mb-0 ml-0 mr-0 mt-0 flex flex-col items-center justify-end px-3 pb-1 pt-5 transition-opacity duration-300 ease-in-out`,
+          isOverlayVisible ? 'opacity-100' : 'opacity-0'
+        )}
+        onMouseMove={() => {
+          setOverlayVisible(true)
+          scheduleOverlayInvisibility()
+        }}
+        style={{
+          // width,
+          // height,
+          boxShadow: 'rgba(0, 0, 0, 1) 0px -77px 100px 15px inset',
+        }}
+      >
+        {/* bottom slider and button bar */}
+        <div
+          className={cn(
+            `flex w-full flex-col items-center justify-center self-end`
+          )}
+        >
+          {/* the (optional) timeline slider bar */}
+          <div
+            className={cn(
+              `flex h-0.5 w-full flex-row items-center bg-gray-100/50`
+            )}
+          >
+            <div
+              className={cn(`flex h-full flex-row items-center`, {
+                'bg-yellow-500/100': meta.isInteractive,
+                'bg-red-500/100': meta.isLive,
+              })}
+              style={{
+                width: '100%', // <-- TODO: compute the % of progression within the experience
+              }}
+            ></div>
+          </div>
+
+          {/* button bar */}
+          <div
+            className={cn(
+              `pointer-events-auto flex h-14 w-full flex-none flex-row items-center justify-between`
+            )}
+          >
+            {/* left-side buttons */}
+            <div
+              className={cn(
+                `flex h-full flex-none items-center justify-center`
+              )}
+            >
+              <TogglePlayback
+                isToggledOn={isPlaying}
+                onClick={togglePlayback}
+              />
+              <StaticOrInteractiveTag
+                isInteractive={meta.isInteractive}
+                size="md"
+                className=""
+              />
+            </div>
+
+            {/* right-side buttons */}
+            <div
+              className={cn(
+                `flex h-full w-32 flex-none flex-row items-center justify-center space-x-2`
+              )}
+            ></div>
+          </div>
+        </div>
+      </div>
+    </>
+  )
+}

packages/app/src/app/embed/StaticOrInteractive.tsx

@@ -0,0 +1,39 @@
+import { PiLightningFill } from 'react-icons/pi'
+
+import { cn } from '@/lib/utils/cn'
+
+export function StaticOrInteractiveTag({
+  isInteractive = false,
+  size = 'md',
+  className = '',
+}: {
+  isInteractive?: boolean
+  size?: 'sm' | 'md'
+  className?: string
+}) {
+  const isStatic = !isInteractive
+
+  return (
+    <div
+      className={cn(
+        `flex flex-none flex-row items-center justify-center border font-medium uppercase`,
+        {
+          'rounded-xs space-x-0.5 py-0.5 pl-0.5 pr-1 text-2xs': size === 'sm',
+          'space-x-1 rounded py-1 pl-1 pr-2 text-xs': size === 'md',
+          'border-yellow-600 text-yellow-600': isInteractive,
+          // " text-red-500 border-red-500": isLive,
+          'border-neutral-600 text-neutral-600': isStatic,
+        },
+        className
+      )}
+    >
+      <PiLightningFill />
+      <span className="-mb-[1px]">
+        {isInteractive
+          ? 'Interactive'
+          : // : isLive ? "Live"
+            'Static content'}
+      </span>
+    </div>
+  )
+}

packages/app/src/app/embed/TogglePlayback.tsx

@@ -0,0 +1,28 @@
+import React from 'react'
+import { IoMdPause, IoMdPlay } from 'react-icons/io'
+
+import { IconSwitch } from '@/components/monitor/icons/icon-switch'
+
+export function TogglePlayback({
+  className = '',
+  isToggledOn,
+  onClick,
+}: {
+  className?: string
+  isToggledOn?: boolean
+  onClick?: () => void
+}) {
+  return (
+    <IconSwitch
+      isToggledOn={isToggledOn}
+      onIcon={IoMdPause}
+      offIcon={IoMdPlay}
+      onClick={onClick}
+      size="md"
+      disabled={false}
+      isAlt={false}
+      thickOnHover={false}
+      className={className}
+    />
+  )
+}

packages/app/src/app/embed/page.tsx

@@ -4,7 +4,7 @@ import { useEffect, useState } from 'react'
 import Head from 'next/head'
 import Script from 'next/script'
 
-import { Embed } from './embed'
+import { ClapperIntegrationMode, Main } from '../main'
 
 // https://nextjs.org/docs/pages/building-your-application/optimizing/fonts
 
@@ -44,7 +44,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
           style={{ display: 'none', visibility: 'hidden' }}
         ></iframe>
       </noscript>
-      <main>{isLoaded && <Embed />}</main>
+      <main>{isLoaded && <Main mode={ClapperIntegrationMode.IFRAME} />}</main>
     </>
   )
 }

packages/app/src/app/embed/useParentController.ts

@@ -0,0 +1,59 @@
+'use client'
+
+import { create } from 'zustand'
+import { Bellhop } from 'bellhop-iframe'
+
+export const useParentController = create<{
+  bellhop: Bellhop
+
+  /**
+   * Whether the communication pipeline seems to be working or not
+   *
+   * Initially we assume that it is, but this can be invalidated
+   * if we know for certain it is not, or in case of exception
+   */
+  canUseBellhop: boolean
+
+  /**
+   * Whether Clapper is ready to receive instructions
+   */
+  hasLoadedBellhop: boolean
+
+  setCanUseBellhop: (canUseBellhop: boolean) => void
+  setHasLoadedBellhop: (hasLoadedBellhop: boolean) => void
+  onMessage: (name: string, callback: Function, priority?: number) => void
+  sendMessage: (type: string, data?: any) => void
+}>((set, get) => ({
+  bellhop: undefined as unknown as Bellhop,
+  canUseBellhop: true,
+  hasLoadedBellhop: false,
+  setCanUseBellhop: (canUseBellhop: boolean) => {
+    set({
+      canUseBellhop,
+    })
+  },
+  setHasLoadedBellhop: (hasLoadedBellhop: boolean) => {
+    set({
+      bellhop: hasLoadedBellhop
+        ? new Bellhop()
+        : (undefined as unknown as Bellhop),
+      hasLoadedBellhop,
+    })
+  },
+  onMessage: (name: string, callback: Function, priority?: number) => {
+    const { bellhop } = get()
+    try {
+      bellhop.on(name, callback, priority)
+    } catch (err) {
+      console.log(`failed to subscribe to parent iframe messages:`, err)
+    }
+  },
+  sendMessage: (type: string, data?: any) => {
+    const { bellhop } = get()
+    try {
+      bellhop.send(type, data)
+    } catch (err) {
+      console.log(`failed to send a message to parent iframe:`, err)
+    }
+  },
+}))