Frame extractor: support custom GlObjectsProvider

Add Configuration.Builder.setGlObjectsProvider to accept
custom GlObjectsProvider.

PiperOrigin-RevId: 784490477

Author: ychaparov

libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java

@@ -33,10 +33,16 @@
 import android.app.Instrumentation;
 import android.content.Context;
 import android.graphics.Bitmap;
+import androidx.media3.common.GlObjectsProvider;
+import androidx.media3.common.GlTextureInfo;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.PlaybackException;
 import androidx.media3.common.util.ConditionVariable;
 import androidx.media3.common.util.NullableType;
+import androidx.media3.effect.DefaultGlObjectsProvider;
+import androidx.media3.effect.GlEffect;
+import androidx.media3.effect.GlShaderProgram;
+import androidx.media3.effect.PassthroughShaderProgram;
 import androidx.media3.effect.Presentation;
 import androidx.media3.exoplayer.ExoPlaybackException;
 import androidx.media3.transformer.ExperimentalFrameExtractor.Frame;
@@ -525,4 +531,40 @@ public void extractFrame_oneFrame_decodesReferenceFramesOnly() throws Exception
                 .skippedInputBufferCount)
         .isEqualTo(13);
   }
+
+  @Test
+  public void extractFrame_withGlObjectsProvider_usesCustomObjectsProvider() throws Exception {
+    GlObjectsProvider customObjectsProvider = new DefaultGlObjectsProvider();
+    AtomicReference<GlObjectsProvider> glObjectsProviderUsedByEffects = new AtomicReference<>();
+    frameExtractor =
+        new ExperimentalFrameExtractor(
+            context,
+            new ExperimentalFrameExtractor.Configuration.Builder()
+                .setGlObjectsProvider(customObjectsProvider)
+                .build());
+    frameExtractor.setMediaItem(
+        MediaItem.fromUri(FILE_PATH),
+        /* effects= */ ImmutableList.of(
+            new GlEffect() {
+              @Override
+              public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) {
+                return new PassthroughShaderProgram() {
+                  @Override
+                  public void queueInputFrame(
+                      GlObjectsProvider glObjectsProvider,
+                      GlTextureInfo inputTexture,
+                      long presentationTimeUs) {
+                    glObjectsProviderUsedByEffects.set(glObjectsProvider);
+                    super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs);
+                  }
+                };
+              }
+            }));
+
+    ListenableFuture<Frame> frameFuture = frameExtractor.getFrame(/* positionMs= */ 8_500);
+    Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS);
+
+    assertThat(frame.presentationTimeMs).isEqualTo(8_531);
+    assertThat(glObjectsProviderUsedByEffects.get()).isEqualTo(customObjectsProvider);
+  }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java

@@ -58,18 +58,22 @@
 import androidx.media3.common.PlaybackException;
 import androidx.media3.common.Player;
 import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.VideoGraph;
 import androidx.media3.common.util.ConditionVariable;
 import androidx.media3.common.util.GlProgram;
 import androidx.media3.common.util.GlUtil;
 import androidx.media3.common.util.NullableType;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
+import androidx.media3.effect.DefaultGlObjectsProvider;
+import androidx.media3.effect.DefaultVideoFrameProcessor;
 import androidx.media3.effect.GlEffect;
 import androidx.media3.effect.GlShaderProgram;
 import androidx.media3.effect.MatrixTransformation;
 import androidx.media3.effect.PassthroughShaderProgram;
 import androidx.media3.effect.RgbMatrix;
 import androidx.media3.effect.ScaleAndRotateTransformation;
+import androidx.media3.effect.SingleInputVideoGraph;
 import androidx.media3.exoplayer.DecoderCounters;
 import androidx.media3.exoplayer.DecoderReuseEvaluation;
 import androidx.media3.exoplayer.ExoPlaybackException;
@@ -83,6 +87,8 @@
 import androidx.media3.exoplayer.source.DefaultMediaSourceFactory;
 import androidx.media3.exoplayer.source.MediaSource;
 import androidx.media3.exoplayer.video.MediaCodecVideoRenderer;
+import androidx.media3.exoplayer.video.PlaybackVideoGraphWrapper;
+import androidx.media3.exoplayer.video.VideoFrameReleaseControl;
 import androidx.media3.exoplayer.video.VideoRendererEventListener;
 import androidx.media3.extractor.DefaultExtractorsFactory;
 import com.google.common.collect.ImmutableList;
@@ -135,6 +141,7 @@ public static final class Builder {
       private SeekParameters seekParameters;
       private MediaCodecSelector mediaCodecSelector;
       private boolean extractHdrFrames;
+      @Nullable private GlObjectsProvider glObjectsProvider;
 
       /** Creates a new instance with default values. */
       public Builder() {
@@ -196,9 +203,24 @@ public Builder setExtractHdrFrames(boolean extractHdrFrames) {
         return this;
       }
 
+      /**
+       * Sets the {@link GlObjectsProvider} to be used by the effect processing pipeline.
+       *
+       * <p>By default, a {@link DefaultGlObjectsProvider} is used.
+       *
+       * @param glObjectsProvider The {@link GlObjectsProvider}.
+       * @return This builder.
+       */
+      @CanIgnoreReturnValue
+      public Builder setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {
+        this.glObjectsProvider = glObjectsProvider;
+        return this;
+      }
+
       /** Builds a new {@link Configuration} instance. */
       public Configuration build() {
-        return new Configuration(seekParameters, mediaCodecSelector, extractHdrFrames);
+        return new Configuration(
+            seekParameters, mediaCodecSelector, extractHdrFrames, glObjectsProvider);
       }
     }
 
@@ -211,13 +233,18 @@ public Configuration build() {
     /** Whether extracting HDR frames is requested. */
     public final boolean extractHdrFrames;
 
+    /** The {@link GlObjectsProvider}. */
+    @Nullable public final GlObjectsProvider glObjectsProvider;
+
     private Configuration(
         SeekParameters seekParameters,
         MediaCodecSelector mediaCodecSelector,
-        boolean extractHdrFrames) {
+        boolean extractHdrFrames,
+        @Nullable GlObjectsProvider glObjectsProvider) {
       this.seekParameters = seekParameters;
       this.mediaCodecSelector = mediaCodecSelector;
       this.extractHdrFrames = extractHdrFrames;
+      this.glObjectsProvider = glObjectsProvider;
     }
   }
 
@@ -294,7 +321,8 @@ public ExperimentalFrameExtractor(Context context, Configuration configuration)
                           context,
                           configuration.mediaCodecSelector,
                           videoRendererEventListener,
-                          /* toneMapHdrToSdr= */ !configuration.extractHdrFrames)
+                          /* toneMapHdrToSdr= */ !configuration.extractHdrFrames,
+                          configuration.glObjectsProvider)
                     },
                 mediaSourceFactory)
             .setSeekParameters(configuration.seekParameters)
@@ -650,6 +678,7 @@ private void ensureConfigured(GlObjectsProvider glObjectsProvider, int width, in
   /** A custom MediaCodecVideoRenderer that renders only one frame per position reset. */
   private final class FrameExtractorRenderer extends MediaCodecVideoRenderer {
     private final boolean toneMapHdrToSdr;
+    @Nullable private final GlObjectsProvider glObjectsProvider;
 
     private boolean frameRenderedSinceLastPositionReset;
     private List<Effect> effectsFromPlayer;
@@ -659,7 +688,8 @@ public FrameExtractorRenderer(
         Context context,
         MediaCodecSelector mediaCodecSelector,
         VideoRendererEventListener videoRendererEventListener,
-        boolean toneMapHdrToSdr) {
+        boolean toneMapHdrToSdr,
+        @Nullable GlObjectsProvider glObjectsProvider) {
       super(
           new Builder(context)
               .setMediaCodecSelector(mediaCodecSelector)
@@ -668,9 +698,27 @@ public FrameExtractorRenderer(
               .setEventListener(videoRendererEventListener)
               .setMaxDroppedFramesToNotify(0));
       this.toneMapHdrToSdr = toneMapHdrToSdr;
+      this.glObjectsProvider = glObjectsProvider;
       effectsFromPlayer = ImmutableList.of();
     }
 
+    @Override
+    protected PlaybackVideoGraphWrapper createPlaybackVideoGraphWrapper(
+        Context context, VideoFrameReleaseControl videoFrameReleaseControl) {
+      if (glObjectsProvider == null) {
+        return super.createPlaybackVideoGraphWrapper(context, videoFrameReleaseControl);
+      }
+      DefaultVideoFrameProcessor.Factory.Builder videoFrameProcessorFactoryBuilder =
+          new DefaultVideoFrameProcessor.Factory.Builder().setGlObjectsProvider(glObjectsProvider);
+      VideoGraph.Factory videoGraphFactory =
+          new SingleInputVideoGraph.Factory(videoFrameProcessorFactoryBuilder.build());
+      return new PlaybackVideoGraphWrapper.Builder(context, videoFrameReleaseControl)
+          .setEnablePlaylistMode(true)
+          .setClock(getClock())
+          .setVideoGraphFactory(videoGraphFactory)
+          .build();
+    }
+
     @Override
     protected void onStreamChanged(
         Format[] formats,