TypeScript Types
Complete TypeScript type definitions for the MP4E SDK.
Overview
Complete TypeScript type definitions for the MP4E SDK. These types can be used for type checking and IDE autocompletion when working with MP4E metadata and player.
Installation
TypeScript types are included in the
@mp4e/react package. No separate installation required.MP4EMetadata
Root metadata types
1// Root metadata structure2interface MP4EMetadata {3 schemaVersion: string; // "2.0"4 metadataType: string; // "mp4e"56 videoDetails: VideoDetails;7 processingStats?: ProcessingStats;89 objects: {10 registry: ObjectRegistry; // Keyed by objectId11 timeline: FrameData[]; // Frame-by-frame positions12 };1314 overlays: Overlay[];15 rules: UnifiedRule[];16 layers: Layer[];17 variables: Variable[];1819 plugins: {20 enabled: string[]; // Plugin types to enable21 config: Record<string, any>;22 };2324 objectGroups?: ObjectGroup[];25 scenes?: Scene[];2627 videoId?: string;28 videoMetadata?: VideoMetadata;29 playerConfig?: PlayerConfig;30 trackingEvents?: TrackingEventsConfig;31 seo?: SEOMetadata;32}3334interface VideoDetails {35 totalFrames?: number | null;36 totalFramesAnalyzed?: number | null;37 fps?: number;38 width?: number;39 height?: number;40 duration?: number;41}4243interface SEOMetadata {44 title: string;45 description: string;46 tags: string[];47 chapters: Chapter[];48 thumbnail?: {49 type: 'frame' | 'custom';50 frame?: number;51 data?: string; // Base64 JPEG for custom52 };53}5455interface Chapter {56 id: string;57 title: string;58 description?: string;59 startTime: number;60 endTime: number;61}Object Types
Object types
1// Object registry (keyed by objectId)2type ObjectRegistry = Record<string, ObjectData>;34interface ObjectData {5 label: string; // AI-detected label6 userLabel?: string; // User-defined display name7 confidence: number; // Detection confidence (0-1)89 // Tracking type10 trackingType?: 'bbox' | 'polygon' | 'surface';11 referencePolygon?: number[]; // Flattened coords for polygon tracking12 surfaceData?: SurfaceData; // For surface tracking1314 // Frame range15 firstSeenFrame?: number;16 lastSeenFrame?: number;1718 // Quality metrics19 trackQuality?: number;20 totalAppearances?: number;21 occlusionRate?: number;2223 // E-commerce24 buyLink?: string;25 productId?: string;26 price?: number;27 currency?: string;28 description?: string;2930 // AI analysis31 llmAnalysis?: {32 style?: string;33 color?: string;34 material?: string;35 setting?: string;36 };3738 // Custom data39 customData?: Record<string, any>;40}4142interface SurfaceData {43 referenceFrame: number;44 referenceCorners: SurfaceCorners;45 trackingQuality?: 'draft' | 'standard' | 'high';46}4748interface SurfaceCorners {49 topLeft: { x: number; y: number };50 topRight: { x: number; y: number };51 bottomRight: { x: number; y: number };52 bottomLeft: { x: number; y: number };53}5455// Timeline frame data56interface FrameData {57 frame: number;58 time: number; // frame / fps59 objects: FrameObject[];60}6162interface FrameObject {63 id: string; // Reference to ObjectData64 bboxXyxy: [number, number, number, number]; // [x1, y1, x2, y2] normalized65 visibility?: number; // 0-166 trackingConfidence?: number; // Per-frame confidence6768 // For polygon tracking69 polygon?: number[]; // Flattened coordinates7071 // For surface tracking72 corners?: SurfaceCorners;73 cornerConfidence?: number;74}7576// Object groups77interface ObjectGroup {78 id: string;79 name: string;80 objectIds: string[];81 displaySettings?: DisplaySettings;82 rules?: UnifiedRule[];83}8485interface DisplaySettings {86 type: string; // e.g., "core:product-card"87 enabled: boolean;88 hoverEnabled?: boolean;89 clickEnabled?: boolean;90 config?: Record<string, any>;91}Variable Types
Variable types (all 15)
1// All 16 variable types2type VariableType =3 | 'text'4 | 'number'5 | 'boolean'6 | 'timer'7 | 'counter'8 | 'date'9 | 'state'10 | 'list'11 | 'object'12 | 'map'13 | 'set'14 | 'json'15 | 'computed'16 | 'accumulated'17 | 'mapped'18 | 'sequence';1920// Base variable structure21interface VariableBase {22 id: string;23 name: string;24 type: VariableType;25 description?: string;26 category?: string;27 hidden?: boolean;28 onChange?: Rule[];29}3031// Text variable32interface TextVariable extends VariableBase {33 type: 'text';34 initialValue: string;35 validation?: {36 minLength?: number;37 maxLength?: number;38 pattern?: string;39 };40}4142// Number variable43interface NumberVariable extends VariableBase {44 type: 'number';45 initialValue: number;46 min?: number;47 max?: number;48 step?: number;49}5051// Boolean variable52interface BooleanVariable extends VariableBase {53 type: 'boolean';54 initialValue: boolean;55}5657// Timer variable58interface TimerVariable extends VariableBase {59 type: 'timer';60 initialValue: number; // Starting value in ms61 direction: 'up' | 'down';62 target?: number; // Target value for conditions63 autoStart?: boolean;64 pauseWithVideo?: boolean;65}6667// Counter variable68interface CounterVariable extends VariableBase {69 type: 'counter';70 initialValue: number;71 min?: number;72 max?: number;73 step?: number;74 overflow?: 'clamp' | 'wrap' | 'error';75}7677// State machine variable78interface StateVariable extends VariableBase {79 type: 'state';80 initialState: string;81 states: string[];82 transitions: StateTransition[];83}8485interface StateTransition {86 from: string;87 to: string;88 condition?: Condition;89}9091// Date variable92interface DateVariable extends VariableBase {93 type: 'date';94 initialValue: string | number | null;95 format?: string;96 timezone?: string; // IANA timezone identifier97}9899// List/Array variable100interface ListVariable extends VariableBase {101 type: 'list';102 initialValue: any[];103 itemType?: 'string' | 'number' | 'object';104 maxLength?: number;105 unique?: boolean;106}107108// Object variable109interface ObjectVariable extends VariableBase {110 type: 'object';111 initialValue: Record<string, any>;112 schema: {113 fields: Array<{114 key: string;115 type: 'string' | 'number' | 'boolean';116 required?: boolean;117 default?: any;118 label?: string;119 }>;120 };121}122123// Map variable124interface MapVariable extends VariableBase {125 type: 'map';126 initialValue: Record<string, any>;127 keyType: 'string';128 valueType: 'string' | 'number' | 'boolean';129}130131// Set variable132interface SetVariable extends VariableBase {133 type: 'set';134 initialValue: any[];135 itemType: 'string' | 'number';136 maxSize?: number;137}138139// JSON variable140interface JsonVariable extends VariableBase {141 type: 'json';142 initialValue: any;143}144145// Computed variable146interface ComputedVariable extends VariableBase {147 type: 'computed';148 expression: string;149 dependencies: string[];150 resultType?: 'string' | 'number' | 'boolean';151 fallback?: any;152}153154// Accumulated variable155interface AccumulatedVariable extends VariableBase {156 type: 'accumulated';157 sourceVariable: string;158 aggregation: 'sum' | 'avg' | 'min' | 'max' | 'count';159 windowSize?: number;160 sampleRate?: number;161}162163// Mapped variable164interface MappedVariable extends VariableBase {165 type: 'mapped';166 sourceVariable: string;167 segments: MappedSegment[];168 defaultOutput?: any;169 easing?: 'linear' | 'easeIn' | 'easeOut' | 'easeInOut';170}171172interface MappedSegment {173 from: number;174 to: number;175 output: any;176}177178// Sequence variable — ordered entry lookup against a source value.179// Pure: the value depends only on the source, never on playback history.180interface SequenceVariable extends VariableBase {181 type: 'sequence';182 /** Variable id/name, or a single wrapped expression like '{{currentFrame}}' */183 source: string;184 /** 'range' compares from/to numerically; 'equals' compares scalars */185 match: 'range' | 'equals';186 /** Ordered — first match wins; overlaps are legal */187 entries: SequenceEntry[];188 /** Value when no entry matches (default null) */189 fallback?: any;190}191192interface SequenceEntry {193 /** Range mode: inclusive lower bound */194 from?: number;195 /** Range mode: inclusive upper bound */196 to?: number;197 /** Equals mode: the scalar to match */198 equals?: any;199 /** Any JSON. Interpolated on selection, including nested object fields */200 value: any;201}202203// Union type204type Variable =205 | TextVariable206 | NumberVariable207 | BooleanVariable208 | TimerVariable209 | CounterVariable210 | DateVariable211 | StateVariable212 | ListVariable213 | ObjectVariable214 | MapVariable215 | SetVariable216 | JsonVariable217 | ComputedVariable218 | AccumulatedVariable219 | MappedVariable220 | SequenceVariable;Rule Types
Rule types (all 21 condition types)
1// Unified rule system2interface UnifiedRule {3 id: string;4 name?: string;5 enabled: boolean;6 priority: number; // Higher = evaluated first78 trigger?: UnifiedTrigger;9 conditions: RuleGroup | Condition;10 onMatch?: Action[];11 onUnmatch?: Action[];1213 evaluateOn?: ('frame' | 'event' | 'stateChange')[];14 cacheResult?: boolean;15 cacheTTL?: number;16 description?: string;17}1819// Condition group20interface RuleGroup {21 logic: 'ALL' | 'ANY' | 'NONE';22 conditions: (RuleGroup | Condition)[];23}2425// All 21 condition types26type ConditionType =27 | 'variable'28 | 'timerVariable'29 | 'counterVariable'30 | 'stateMachineVariable'31 | 'arrayVariable'32 | 'state'33 | 'event'34 | 'eventData'35 | 'elementStats'36 | 'playbackStatus'37 | 'time'38 | 'frame'39 | 'object'40 | 'userAttribute'41 | 'abTest'42 | 'device'43 | 'geo'44 | 'schedule'45 | 'pluginState'46 | 'expression'47 | 'inputKey';4849// Comparison operators50type ComparisonOperator =51 | 'eq' | 'neq'52 | 'gt' | 'gte' | 'lt' | 'lte'53 | 'in' | 'notIn'54 | 'contains' | 'startsWith' | 'endsWith'55 | 'matches'56 | 'exists' | 'notExists'57 | 'between';5859// Base condition60interface ConditionBase {61 type: ConditionType;62 operator?: ComparisonOperator;63 value?: any;64 value2?: any; // For 'between' operator65}6667// Variable condition68interface VariableCondition extends ConditionBase {69 type: 'variable';70 name: string;71}7273// Time condition74interface TimeCondition extends ConditionBase {75 type: 'time';76 timeType: 'playback' | 'wallClock' | 'sessionDuration';77}7879// Object condition80interface ObjectCondition extends ConditionBase {81 type: 'object';82 objectId?: string;83 groupId?: string;84 check: 'isVisible' | 'anyVisible' | 'confidence';85}8687// Device condition88interface DeviceCondition extends ConditionBase {89 type: 'device';90 deviceType: 'mobile' | 'tablet' | 'desktop';91}9293// Element stats condition94interface ElementStatsCondition extends ConditionBase {95 type: 'elementStats';96 elementId: string;97 stat: 'hoverCount' | 'clickCount' | 'dwellTimeMs' | 'firstInteractionAt' | 'lastInteractionAt';98}99100// Playback status condition101interface PlaybackStatusCondition extends ConditionBase {102 type: 'playbackStatus';103 check: 'isPlaying' | 'isPaused' | 'isMuted' | 'volume' | 'playbackRate';104}105106// Expression condition107interface ExpressionCondition extends ConditionBase {108 type: 'expression';109 expression: string;110}111112// Union type (simplified - add more as needed)113type Condition =114 | VariableCondition115 | TimeCondition116 | ObjectCondition117 | DeviceCondition118 | ElementStatsCondition119 | PlaybackStatusCondition120 | ExpressionCondition121 | ConditionBase;Overlay Types
Overlay types
1// Overlay type2interface Overlay {3 id: string;4 name: string;5 type: string; // Plugin type (e.g., "core:button")6 zIndex: number;7 opacity: number;89 // Positioning10 position: OverlayPosition;11 size?: OverlaySize;1213 // Visibility14 visibility: OverlayVisibility;1516 // Plugin configuration17 config: Record<string, any>;1819 // Object binding (for attached overlays)20 objectBinding?: OverlayObjectBinding;2122 // Engine-resolved crop of the playing video, drawn beneath this overlay's23 // plugin. Independent of objectBinding: the overlay can be positioned by one24 // object while its crop follows another.25 videoRegion?: VideoRegion;2627 // Animation28 animation?: OverlayAnimation;2930 // Events31 eventHandlers?: Record<string, Action[]>;32 eventRules?: Record<string, Rule[]>;33}3435interface VideoRegion {36 source: VideoRegionSource;37 shape?: 'circle' | 'roundedRect' | 'none';38 radius?: number | string;39 fit?: 'cover' | 'contain';40 opacity?: number | string;41 /** Soft edge in px; 0 = hard edge */42 feather?: number | string;43 /** Named params, not a CSS filter string — native players map these too */44 filter?: {45 brightness?: number | string;46 contrast?: number | string;47 saturate?: number | string;48 blur?: number | string;49 invert?: number | string;50 grayscale?: number | string;51 };52 border?: { width?: number | string; color?: string };53}5455// Every field accepts a string as well as a number: a templated value56// (e.g. "{{Shot.sx}}") resolves to a string, which the engine coerces.57type VideoRegionSource =58 | { mode: 'rect'; x: number | string; y: number | string;59 w: number | string; h: number | string }60 | { mode: 'object'; objectId: string; padding?: number | string };6162interface OverlayPosition {63 type: 'fixed' | 'attached' | 'absolute';6465 // For fixed/absolute66 x?: number;67 y?: number;68 anchor?: 'top-left' | 'top-center' | 'top-right' |69 'center-left' | 'center' | 'center-right' |70 'bottom-left' | 'bottom-center' | 'bottom-right';7172 // For attached (to object)73 objectId?: string;74 offset?: { x: number; y: number };75 followObject?: boolean;76}7778interface OverlaySize {79 width: number | 'auto';80 height: number | 'auto';81 minWidth?: number;82 maxWidth?: number;83 minHeight?: number;84 maxHeight?: number;85 aspectRatio?: number;86 scaleWithVideo?: boolean;87}8889interface OverlayVisibility {90 fromFrame?: number;91 toFrame?: number;92 showWhen?: Condition | RuleGroup;93 hideWhen?: Condition | RuleGroup;94 rules?: Rule[];95}9697interface OverlayObjectBinding {98 objectId: string;99 position: 'above' | 'below' | 'left' | 'right' | 'center';100 offset?: { x: number; y: number };101 showOnHover?: boolean;102 showOnClick?: boolean;103 followPosition?: boolean;104}105106interface OverlayAnimation {107 enter?: AnimationConfig;108 exit?: AnimationConfig;109 idle?: AnimationConfig;110}111112interface AnimationConfig {113 type: 'fade' | 'slide' | 'scale' | 'bounce' | 'none';114 duration?: number;115 delay?: number;116 easing?: string;117 direction?: 'up' | 'down' | 'left' | 'right';118}119120// Layer structure121interface Layer {122 id: string;123 name: string;124 order: number;125 enabled: boolean;126 blendMode: 'merge' | 'replace' | 'exclusive';127128 overlays: Overlay[];129 rules?: LayerRule;130 visibilityRules?: LayerVisibilityRule;131132 locked?: boolean;133 color?: string;134 collapsed?: boolean;135}136137interface LayerVisibilityRule {138 logic: 'ALL' | 'ANY' | 'NONE';139 conditions: (RuleGroup | Condition)[];140}Action Types
Action types (69 total)
1// Common action types (69 total — see Actions Reference for complete list)2type ActionType =3 | 'seek'4 | 'play'5 | 'pause'6 | 'goToScene'7 | 'openUrl'8 | 'navigate'9 | 'showOverlay'10 | 'hideOverlay'11 | 'toggleOverlay'12 | 'setVariable'13 | 'setState'14 | 'timerControl'15 | 'counterControl'16 | 'arrayControl'17 | 'trackEvent'18 | 'emitEvent'19 | 'pluginAction'20 | 'showNotification'21 | 'showTracking'22 | 'hideTracking'23 | 'setPlaybackRate'24 | 'setVolume'25 | 'mute'26 | 'unmute'27 | 'wait'28 | 'showCustomModal'29 | 'closeCustomModal'30 | 'setOverlayStyle'31 | string; // 69 types total — extensible3233// Base action34interface ActionBase {35 type: ActionType;36 delay?: number; // Delay before execution (ms)37 condition?: Condition; // Only execute if condition met38}3940// Seek action41interface SeekAction extends ActionBase {42 type: 'seek';43 time?: number; // Seconds44 frame?: number; // Frame number45 relative?: boolean; // Relative to current position46}4748// Open URL action49interface OpenUrlAction extends ActionBase {50 type: 'openUrl';51 url: string;52 target?: 'blank' | 'self' | 'parent' | 'top';53}5455// Overlay actions56interface ShowOverlayAction extends ActionBase {57 type: 'showOverlay';58 overlayId: string;59}6061interface HideOverlayAction extends ActionBase {62 type: 'hideOverlay';63 overlayId: string;64}6566// Playback actions67interface PauseAction extends ActionBase {68 type: 'pause';69 reason?: string;70}7172interface PlayAction extends ActionBase {73 type: 'play';74}7576interface SetPlaybackRateAction extends ActionBase {77 type: 'setPlaybackRate';78 rate: number; // 0.25 - 2.079}8081interface SetVolumeAction extends ActionBase {82 type: 'setVolume';83 volume: number; // 0.0 - 1.084}8586// State action87interface SetStateAction extends ActionBase {88 type: 'setState';89 key: string;90 value: any; // Can be expression string91}9293// Track event action94interface TrackEventAction extends ActionBase {95 type: 'trackEvent';96 eventName: string;97 properties?: Record<string, any>;98}99100// Highlight object action101interface HighlightObjectAction extends ActionBase {102 type: 'highlightObject';103 objectId: string;104 duration?: number; // ms105 style?: {106 color?: string;107 borderWidth?: number;108 animation?: string;109 };110}111112// Show notification action113interface ShowNotificationAction extends ActionBase {114 type: 'showNotification';115 message: string;116 variant?: 'info' | 'success' | 'warning' | 'error';117 duration?: number; // ms (0 = persistent)118 position?: 'top' | 'bottom' | 'top-left' | 'top-right' | 'bottom-left' | 'bottom-right';119}120121// Plugin action122interface CallPluginAction extends ActionBase {123 type: 'callPlugin';124 plugin: string; // pluginName.methodName125 params?: Record<string, any>;126}127128// Union type129type Action =130 | SeekAction131 | OpenUrlAction132 | ShowOverlayAction133 | HideOverlayAction134 | PauseAction135 | PlayAction136 | SetPlaybackRateAction137 | SetVolumeAction138 | SetStateAction139 | TrackEventAction140 | HighlightObjectAction141 | ShowNotificationAction142 | CallPluginAction143 | { type: 'mute' }144 | { type: 'unmute' };Player Types
Player props and ref
1// Player component props2interface MP4EPlayerProps {3 // Source4 src: string;5 metadata?: MP4EMetadata | string;67 // Playback8 autoplay?: boolean;9 muted?: boolean;10 loop?: boolean;11 preload?: 'none' | 'metadata' | 'auto';12 playbackRate?: number;13 startTime?: number;1415 // Display16 width?: number | string;17 height?: number | string;18 aspectRatio?: '16:9' | '4:3' | '1:1' | 'auto';19 objectFit?: 'contain' | 'cover' | 'fill';20 poster?: string;2122 // Features23 interactive?: boolean;24 showObjects?: boolean;25 showOverlays?: boolean;26 showControls?: boolean;27 objectGroups?: string[];2829 // Overrides30 overrideVariables?: Record<string, any>;31 overrideDisplaySettings?: DisplaySettings;3233 // Debug34 debug?: boolean;35 showDebugOverlay?: boolean;3637 // Callbacks - Lifecycle38 onReady?: () => void;39 onLoad?: (event: { duration: number }) => void;40 onMetadataLoaded?: (metadata: MP4EMetadata) => void;41 onEngineReady?: (engine: MP4EEngine) => void;42 onDestroy?: () => void;4344 // Callbacks - Playback45 onPlay?: () => void;46 onPause?: (event: { currentTime: number; reason?: string }) => void;47 onEnded?: () => void;48 onTimeUpdate?: (event: { currentTime: number; currentFrame: number }) => void;49 onSeeking?: (event: { fromTime: number; toTime: number }) => void;50 onSeeked?: (event: { currentTime: number }) => void;51 onBufferStart?: () => void;52 onBufferEnd?: () => void;53 onVolumeChange?: (event: { volume: number }) => void;54 onRateChange?: (event: { playbackRate: number }) => void;55 onFirstPlay?: () => void;5657 // Callbacks - Objects58 onObjectClick?: (event: ObjectEvent) => void;59 onObjectHover?: (event: ObjectEvent) => void;60 onObjectHoverEnd?: (event: ObjectEvent) => void;61 onObjectVisible?: (event: ObjectEvent) => void;62 onObjectHidden?: (event: ObjectEvent) => void;6364 // Callbacks - Overlays65 onOverlayShow?: (event: { overlayId: string }) => void;66 onOverlayHide?: (event: { overlayId: string }) => void;67 onOverlayClick?: (event: { overlayId: string }) => boolean | void;68 onPluginEvent?: (event: { pluginId: string; eventName: string; data: any }) => void;6970 // Callbacks - Variables71 onVariableChange?: (event: { variableName: string; previousValue: any; newValue: any }) => void;7273 // Callbacks - Actions74 onAction?: (action: Action) => boolean | void;75 onTrackEvent?: (event: { eventName: string; properties: Record<string, any> }) => void;7677 // Callbacks - Errors78 onError?: (error: { message: string; code: string; details?: any }) => void;7980 // Styling81 className?: string;82 style?: React.CSSProperties;83}8485interface ObjectEvent {86 objectId: string;87 objectLabel: string;88 confidence: number;89 boundingBox: { x: number; y: number; width: number; height: number };90 objectData?: ObjectData;91 clickX?: number;92 clickY?: number;93}9495// Player ref methods96interface MP4EPlayerRef {97 // Playback control98 play(): void;99 pause(): void;100 seek(time: number): void;101 seekToFrame(frame: number): void;102 setPlaybackRate(rate: number): void;103 setVolume(volume: number): void;104 mute(): void;105 unmute(): void;106107 // State getters108 getCurrentTime(): number;109 getCurrentFrame(): number;110 getDuration(): number;111 getTotalFrames(): number;112 isPaused(): boolean;113 isMuted(): boolean;114 getVolume(): number;115 getPlaybackRate(): number;116117 // Objects118 getVisibleObjects(): ObjectData[];119 getAllObjects(): ObjectRegistry;120 highlightObject(objectId: string, duration?: number): void;121122 // Variables123 getVariable(name: string): any;124 setVariable(name: string, value: any): void;125 resetVariables(): void;126127 // Actions128 executeAction(action: Action): void;129 executeActions(actions: Action[]): void;130131 // Overlays132 showOverlay(overlayId: string): void;133 hideOverlay(overlayId: string): void;134135 // Scenes136 goToScene(sceneId: string): void;137 getCurrentScene(): string | null;138139 // Engine access140 getEngine(): MP4EEngine | null;141 getMetadata(): MP4EMetadata | null;142143 // Video element144 getVideoElement(): HTMLVideoElement | null;145}