magic-script · panwrona · Mar 30, 2020 · Mar 19, 2020 · Mar 20, 2020 · Mar 20, 2020
diff --git a/android/src/main/java/com/magicleap/magicscript/ARComponentManager.java b/android/src/main/java/com/magicleap/magicscript/ARComponentManager.java
@@ -78,11 +78,11 @@
 import com.magicleap.magicscript.scene.nodes.UiTextEditNode;
 import com.magicleap.magicscript.scene.nodes.UiTextNode;
 import com.magicleap.magicscript.scene.nodes.UiTimePickerNode;
-import com.magicleap.magicscript.scene.nodes.audio.AudioFileProvider;
+import com.magicleap.magicscript.scene.nodes.audio.FileProvider;
 import com.magicleap.magicscript.scene.nodes.audio.AudioNode;
 import com.magicleap.magicscript.scene.nodes.audio.ExternalAudioEngine;
 import com.magicleap.magicscript.scene.nodes.audio.GvrAudioEngineWrapper;
-import com.magicleap.magicscript.scene.nodes.audio.UriAudioProvider;
+import com.magicleap.magicscript.scene.nodes.audio.UriFileProvider;
 import com.magicleap.magicscript.scene.nodes.audio.VrAudioEngine;
 import com.magicleap.magicscript.scene.nodes.base.ReactNode;
 import com.magicleap.magicscript.scene.nodes.button.UiButtonNode;
@@ -263,8 +263,9 @@ public void createModelNode(final ReadableMap props, final String nodeId) {
     @ReactMethod
     public void createVideoNode(final ReadableMap props, final String nodeId) {
         mainHandler.post(() -> {
-            VideoPlayer videoPlayer = new VideoPlayerImpl(context);
-            addNode(new VideoNode(props, context, videoPlayer, videoRenderableLoader, videoNodeClipper, arResourcesProvider), nodeId);
+            FileProvider fileProvider = new UriFileProvider(context);
+            VideoPlayer videoPlayer = new VideoPlayerImpl(context, fileProvider);
+            addNode(new VideoNode(props, context, videoPlayer, videoRenderableLoader, viewRenderableLoader, videoNodeClipper, fontProvider, arResourcesProvider), nodeId);
         });
     }
 
@@ -452,8 +453,8 @@ public void createAudioNode(final ReadableMap props, final String nodeId) {
             GvrAudioEngine gvrAudioEngine = new GvrAudioEngine(context, GvrAudioEngine.RenderingMode.BINAURAL_HIGH_QUALITY);
             ExternalAudioEngine externalAudioEngine = new GvrAudioEngineWrapper(gvrAudioEngine);
             VrAudioEngine audioEngine = new VrAudioEngine(Executors.newSingleThreadExecutor(), externalAudioEngine);
-            AudioFileProvider audioFileProvider = new UriAudioProvider(context);
-            AudioNode node = new AudioNode(props, context, audioEngine, audioFileProvider);
+            FileProvider fileProvider = new UriFileProvider(context);
+            AudioNode node = new AudioNode(props, context, audioEngine, fileProvider);
             addNode(node, nodeId);
         });
     }

diff --git a/android/src/main/java/com/magicleap/magicscript/scene/nodes/UiTextNode.kt b/android/src/main/java/com/magicleap/magicscript/scene/nodes/UiTextNode.kt
@@ -158,6 +158,9 @@ open class UiTextNode(
             "right" -> {
                 (view as TextView).gravity = Gravity.RIGHT
             }
+            "bottom-center" -> {
+                (view as TextView).gravity = Gravity.BOTTOM or Gravity.CENTER_HORIZONTAL
+            }
         }
     }
 

diff --git a/android/src/main/java/com/magicleap/magicscript/scene/nodes/audio/AudioNode.kt b/android/src/main/java/com/magicleap/magicscript/scene/nodes/audio/AudioNode.kt
@@ -30,7 +30,7 @@ open class AudioNode(
     initProps: ReadableMap,
     private val context: Context,
     private var audioEngine: AudioEngine,
-    private val fileProvider: AudioFileProvider
+    private val fileProvider: FileProvider
 ) : TransformNode(initProps, false) {
 
     companion object {

diff --git a/...pt/scene/nodes/audio/AudioFileProvider.kt → ...cscript/scene/nodes/audio/FileProvider.kt b/...pt/scene/nodes/audio/AudioFileProvider.kt → ...cscript/scene/nodes/audio/FileProvider.kt
@@ -19,7 +19,7 @@ package com.magicleap.magicscript.scene.nodes.audio
 import android.net.Uri
 import java.io.File
 
-interface AudioFileProvider {
+interface FileProvider {
     fun provideFile(uri: Uri, result: (File) -> Unit)
     fun onDestroy()
 }
diff --git a/...ipt/scene/nodes/audio/UriAudioProvider.kt → ...ript/scene/nodes/audio/UriFileProvider.kt b/...ipt/scene/nodes/audio/UriAudioProvider.kt → ...ript/scene/nodes/audio/UriFileProvider.kt
@@ -24,9 +24,12 @@ import java.io.BufferedInputStream
 import java.io.File
 import java.io.InputStream
 import java.net.URL
-
-class UriAudioProvider(private val context: Context) :
-    AudioFileProvider {
+/*
+ * When the Uri is provided and framework is not handling the Files properly,
+ * we use this provider to fetch the files and use it for example for audio files, subtitle files etc
+ */
+class UriFileProvider(private val context: Context) :
+    FileProvider {
 
     companion object {
         private const val READ_TIMEOUT = 5000
@@ -64,7 +67,7 @@ class UriAudioProvider(private val context: Context) :
                     bufferedInputStream.close()
                     fileDownloaded(uri.path, result, file)
                 } catch (e: Exception) {
-                    logMessage("Error during reading Audio File $e", true)
+                    logMessage("Error during reading file $e", true)
                 }
             }
         )

diff --git a/android/src/main/java/com/magicleap/magicscript/scene/nodes/video/UiSubtitlesNode.kt b/android/src/main/java/com/magicleap/magicscript/scene/nodes/video/UiSubtitlesNode.kt
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2020 Magic Leap, Inc. All Rights Reserved
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.magicleap.magicscript.scene.nodes.video
+
+import android.content.Context
+import android.os.Bundle
+import android.view.LayoutInflater
+import android.view.View
+import android.widget.TextView
+import com.facebook.react.bridge.ReadableMap
+import com.google.ar.sceneform.rendering.Renderable
+import com.magicleap.magicscript.R
+import com.magicleap.magicscript.ar.RenderPriority
+import com.magicleap.magicscript.ar.clip.Clipper
+import com.magicleap.magicscript.ar.renderable.ViewRenderableLoader
+import com.magicleap.magicscript.font.FontProvider
+import com.magicleap.magicscript.scene.nodes.UiTextNode
+
+class UiSubtitlesNode(
+        props: ReadableMap,
+        context: Context,
+        viewRenderableLoader: ViewRenderableLoader,
+        nodeClipper: Clipper,
+        fontProvider: FontProvider
+) : UiTextNode(props, context, viewRenderableLoader, nodeClipper, fontProvider) {
+
+    override fun provideView(context: Context): View {
+        return LayoutInflater.from(context).inflate(R.layout.subtitles, null) as TextView
+    }
+
+    override fun onViewLoaded(viewRenderable: Renderable) {
+        super.onViewLoaded(viewRenderable)
+        viewRenderable.renderPriority = RenderPriority.ABOVE_DEFAULT
+    }
+}
diff --git a/android/src/main/java/com/magicleap/magicscript/scene/nodes/video/VideoNode.kt b/android/src/main/java/com/magicleap/magicscript/scene/nodes/video/VideoNode.kt
@@ -17,7 +17,10 @@
 package com.magicleap.magicscript.scene.nodes.video
 
 import android.content.Context
+import android.net.Uri
 import android.os.Bundle
+import com.facebook.react.bridge.JavaOnlyArray
+import com.facebook.react.bridge.JavaOnlyMap
 import com.facebook.react.bridge.ReadableMap
 import com.google.ar.sceneform.math.Vector3
 import com.google.ar.sceneform.rendering.ExternalTexture
@@ -27,19 +30,26 @@ import com.magicleap.magicscript.ar.clip.Clipper
 import com.magicleap.magicscript.ar.renderable.RenderableLoadRequest
 import com.magicleap.magicscript.ar.renderable.RenderableResult
 import com.magicleap.magicscript.ar.renderable.VideoRenderableLoader
+import com.magicleap.magicscript.ar.renderable.ViewRenderableLoader
+import com.magicleap.magicscript.font.FontProvider
+import com.magicleap.magicscript.scene.NodesManager
+import com.magicleap.magicscript.scene.nodes.UiTextNode
+import com.magicleap.magicscript.scene.nodes.UiTextNode.Companion.PROP_BOUNDS_SIZE
+import com.magicleap.magicscript.scene.nodes.UiTextNode.Companion.PROP_TEXT_ALIGNMENT
 import com.magicleap.magicscript.scene.nodes.base.TransformNode
 import com.magicleap.magicscript.scene.nodes.props.AABB
-import com.magicleap.magicscript.utils.logMessage
-import com.magicleap.magicscript.utils.putDefault
-import com.magicleap.magicscript.utils.readFilePath
+import com.magicleap.magicscript.utils.*
 
 class VideoNode(
     initProps: ReadableMap,
     private val context: Context,
     private val videoPlayer: VideoPlayer,
     private val videoRenderableLoader: VideoRenderableLoader,
+    private val viewRenderableLoader: ViewRenderableLoader,
     private val nodeClipper: Clipper,
+    private val fontProvider: FontProvider,
     private val arResourcesProvider: ArResourcesProvider
+
 ) : TransformNode(initProps, useContentNodeAlignment = true) {
 
     companion object {
@@ -50,12 +60,16 @@ class VideoNode(
         const val PROP_LOOPING = "looping"
         const val PROP_ACTION = "action"
         const val PROP_VOLUME = "volume"
+        const val PROP_TIMED_TEXT_PATH = "timedTextPath"
+        const val PROP_SEEK_TO = "seekTo"
 
         const val ACTION_START = "start"
         const val ACTION_STOP = "stop"
         const val ACTION_PAUSE = "pause"
 
         const val DEFAULT_VOLUME = 1.0
+
+        private const val SUBTITLES_MARGIN_BOTTOM = 0.05 // in meters
     }
 
     var onVideoPreparedListener: (() -> Unit)? = null
@@ -75,6 +89,7 @@ class VideoNode(
     private val initialHeight = 1F // meters
 
     private var lastUserAction: String = ""
+    private var subtitles: UiSubtitlesNode? = null
 
     init {
         // set default values of properties
@@ -91,6 +106,11 @@ class VideoNode(
         setAction(props)
         setLooping(props)
         setVolume(props)
+        setSeekTo(props)
+
+        if (props.containsAny(PROP_SIZE)) {
+            updateSubtitlesProps()
+        }
     }
 
     override fun onVisibilityChanged(visibility: Boolean) {
@@ -133,6 +153,10 @@ class VideoNode(
     // destroying media player when node is detached (e.g. on scene change)
     override fun onDestroy() {
         super.onDestroy()
+        if (subtitles != null) {
+            contentNode.removeChild(subtitles)
+            subtitles = null
+        }
         videoPlayer.release()
         renderableLoadRequest?.let {
             videoRenderableLoader.cancel(it)
@@ -143,6 +167,58 @@ class VideoNode(
         nodeClipper.applyClipBounds(this, clipBounds)
     }
 
+    private fun createSubtitles() {
+        if (subtitles == null) {
+            subtitles = UiSubtitlesNode(
+                props = getSubtitleProps(),
+                context = context,
+                viewRenderableLoader = viewRenderableLoader,
+                nodeClipper = nodeClipper,
+                fontProvider = fontProvider
+            )
+            contentNode.addChild(subtitles)
+            subtitles?.build()
+        }
+    }
+
+    private fun getSubtitleProps(): JavaOnlyMap {
+        val videoSize = readVideoSize(properties) ?: Vector2(initialWidth, initialHeight)
+
+        // we have to apply reverted content scale, so the subtitles are not stretched
+        val contentScale = contentNode.localScale
+        val scaleX = if (contentScale.x > 0) 1f / contentScale.x else 0f
+        val scaleY = if (contentScale.y > 0) 1f / contentScale.y else 0f
+        val scaleZ = if (contentScale.x > 0) 1f / contentScale.z else 0f
+
+        return JavaOnlyMap.of(
+            UiTextNode.PROP_TEXT_SIZE, 0.1,
+            PROP_LOCAL_POSITION, JavaOnlyArray.of(0f, SUBTITLES_MARGIN_BOTTOM, 0.01f),
+            PROP_LOCAL_SCALE, JavaOnlyArray.of(scaleX, scaleY, scaleZ),
+            PROP_ALIGNMENT, "bottom-center",
+            PROP_TEXT_ALIGNMENT, "bottom-center",
+            PROP_BOUNDS_SIZE, JavaOnlyMap.of(
+                PROP_BOUNDS_SIZE, JavaOnlyArray.of(videoSize.x, videoSize.y)
+            )
+        )
+    }
+
+    private fun updateSubtitlesProps() {
+        subtitles?.update(getSubtitleProps())
+    }
+
+    private fun getTimedTextPath(props: Bundle): Uri? {
+        if (props.containsKey(PROP_TIMED_TEXT_PATH)) {
+            return props.readFilePath(PROP_TIMED_TEXT_PATH, context)
+        }
+        return null
+    }
+
+    private fun setSeekTo(props: Bundle) {
+        if (props.containsKey(PROP_SEEK_TO)) {
+            videoPlayer.seekTo(props.getDouble(PROP_SEEK_TO).toInt())
+        }
+    }
+
     private fun loadVideo() {
         if (!arResourcesProvider.isArLoaded()) {
             // ar must be load to create ExternalTexture object
@@ -151,11 +227,38 @@ class VideoNode(
 
         val videoUri = properties.readFilePath(PROP_VIDEO_PATH, context)
         if (videoUri != null) {
+            subtitles?.update(
+                // add empty text to hide last subtitles
+                JavaOnlyMap.of(UiTextNode.PROP_TEXT, "")
+            )
             val texture = ExternalTexture()
             try {
-                videoPlayer.loadVideo(videoUri, texture.surface, onLoadedListener = {
-                    onVideoPreparedListener?.invoke()
-                })
+                val timedTextUri = getTimedTextPath(properties)
+                if (timedTextUri != null) {
+                    videoPlayer.loadVideo(
+                        uri = videoUri,
+                        subtitlesPath = timedTextUri,
+                        onSubtitleChangeListener = {
+                            subtitles?.update(JavaOnlyMap.of(UiTextNode.PROP_TEXT, it))
+                        },
+                        surface = texture.surface,
+                        onLoadedListener = {
+                            onVideoPreparedListener?.invoke()
+                            createSubtitles()
+                        })
+                } else {
+                    if (subtitles != null) {
+                        contentNode.removeChild(subtitles)
+                        subtitles = null
+                    }
+                    videoPlayer.loadVideo(
+                        uri = videoUri,
+                        surface = texture.surface,
+                        onLoadedListener = {
+                            onVideoPreparedListener?.invoke()
+                        })
+                }
+
             } catch (exception: Exception) {
                 logMessage("video load exception: $exception", warn = true)
             }
@@ -183,16 +286,22 @@ class VideoNode(
 
     // changing video size by scaling the content node
     private fun setSize(props: Bundle) {
+        readVideoSize(props)?.let { videoSize ->
+            val scaleX = videoSize.x / initialWidth
+            val scaleY = videoSize.y / initialHeight
+            contentNode.localScale = Vector3(scaleX, scaleY, 1F)
+            applyClipBounds()
+        }
+    }
+
+    private fun readVideoSize(props: Bundle): Vector2? {
         val sizeArray = props.getSerializable(PROP_SIZE) as? ArrayList<Double>
         if (sizeArray != null && sizeArray.size == 2) {
             val widthMeters = sizeArray[0].toFloat()
             val heightMeters = sizeArray[1].toFloat()
-
-            val scaleX = widthMeters / initialWidth
-            val scaleY = heightMeters / initialHeight
-            contentNode.localScale = Vector3(scaleX, scaleY, 1F)
-            applyClipBounds()
+            return Vector2(widthMeters, heightMeters)
         }
+        return null
     }
 
     private fun setAction(props: Bundle) {

diff --git a/android/src/main/java/com/magicleap/magicscript/scene/nodes/video/VideoPlayer.kt b/android/src/main/java/com/magicleap/magicscript/scene/nodes/video/VideoPlayer.kt
@@ -29,7 +29,7 @@ interface VideoPlayer {
     val isReady: Boolean
 
     @Throws(Exception::class)
-    fun loadVideo(uri: Uri, surface: Surface, onLoadedListener: () -> Unit)
+    fun loadVideo(uri: Uri, subtitlesPath: Uri? = null, onSubtitleChangeListener: ((String) -> Unit)? = null, surface: Surface, onLoadedListener: () -> Unit)
 
     @Throws(IllegalStateException::class)
     fun start()
@@ -41,4 +41,8 @@ interface VideoPlayer {
     fun stop()
 
     fun release()
+
+    fun seekTo(millis: Int)
+
+    fun clearTimedTextListener()
 }