illegalstateexception : Failed to add the track to the muxer while replacing audio of video

[pawaom]

I am Trying this code Trying to replace the audio of a Video , both the files have sound

I am getting this error

illegalstateexception : Failed to add the track to the muxer

Also how can we overlay audio keeping video's audio modifying the below code (I want to pass just the URI's, i have referred the demo code , but I have never used databinding and dnot plan to use it a simple implementation example will be helpful)

 public class MainActivity extends AppCompatActivity {
 
private File targetVideoFile;

@Override
protected void onCreate(@Nullable Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    setContentView(R.layout.frag2);
    Uri videouri = Uri.parse("content://media/external/video/media/9092");
    Uri audioUri = Uri.parse("content://media/external/audio/media/213");

    TransformationListener tl = new TransformationListener() {

        @Override
        public void onStarted(String id) {
        // TODO: Implement this method
            Toast.makeText(MainActivity.this, "Started ", Toast.LENGTH_SHORT).show();
        }

        @Override
        public void onProgress(String id, float progress) {
     // TODO: Implement this method
            Toast.makeText(MainActivity.this, "progress " + progress, Toast.LENGTH_SHORT).show();
        }

        @Override
        public void onCompleted(String id, List trackTransformationInfos) {
            //sm("ok");
            Toast.makeText(MainActivity.this, "Completed ", Toast.LENGTH_SHORT).show();
        }

        @Override
        public void onCancelled(String id, List trackTransformationInfos) {
       // TODO: Implement this method
        }

        @Override
        public void onError(String id, Throwable cause, List trackTransformationInfos) {
            // sm("error=" + id);
            Toast.makeText(MainActivity.this, cause.toString(), Toast.LENGTH_SHORT).show();
        }
    };
    Button TransFormButton = (Button) findViewById(R.id.TransformButton);
    TransFormButton.setOnClickListener(view -> {
        Toast.makeText(MainActivity.this, "clicked ", Toast.LENGTH_SHORT).show();
        File targetDirectory = getTargetFileDirectory();
        targetVideoFile = new File(targetDirectory, "transcoded_" + "abc.mp4");
        if (targetVideoFile.exists()) {
            Toast.makeText(MainActivity.this, "existed", Toast.LENGTH_SHORT).show();
            targetVideoFile.delete();
            Toast.makeText(MainActivity.this, "Deleted ", Toast.LENGTH_SHORT).show();
        }
        MediaSource videoMediaSource = null;
        try {
            videoMediaSource = new MediaExtractorMediaSource(MainActivity.this, videouri);
        } catch (MediaSourceException e) {
            throw new RuntimeException(e);
        }
        MediaSource audioMediaSource = null;
        try {
            audioMediaSource = new MediaExtractorMediaSource(MainActivity.this, audioUri);
        } catch (MediaSourceException e) {
            throw new RuntimeException(e);
        }
        
        MediaTarget combinedMediaTarget = null;
        try {
            combinedMediaTarget = new MediaMuxerMediaTarget(targetVideoFile.getAbsolutePath(),
                    2, 0, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
        } catch (MediaTargetException e) {
            throw new RuntimeException(e);
        }
        TrackTransform videoTrackTransform = new TrackTransform.Builder(videoMediaSource, 0, combinedMediaTarget)
                .setDecoder(new MediaCodecDecoder())
                .setEncoder(new MediaCodecEncoder())
                .setTargetTrack(0)
                .setRenderer(new GlVideoRenderer(null))
                .setTargetFormat(null)
                .build();

        TrackTransform audioTrackTransform = new TrackTransform.Builder(audioMediaSource, 0, combinedMediaTarget)
                .setDecoder(new MediaCodecDecoder())
                .setEncoder(new MediaCodecEncoder())
                .setTargetTrack(1)
                .setTargetFormat(null)
                .build();

        List trackTransforms = new ArrayList<>();
        trackTransforms.add(videoTrackTransform);
        trackTransforms.add(audioTrackTransform);
        MediaTransformer mediaTransformer = new MediaTransformer(getApplicationContext());
        mediaTransformer.transform("663600", trackTransforms, tl, MediaTransformer.GRANULARITY_DEFAULT);
    });
}



@NonNull
private File getTargetFileDirectory() {
    File targetDirectory = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES)
            + File.separator
            + "LiTr");
    if (!targetDirectory.exists()) {
        targetDirectory.mkdirs();
    }
    return targetDirectory;
   }
}

LogCat shows this error

Unsupported mime 'audio/mpeg'
2023-01-22 11:31:41.946 5631-8853 TransformationJob com.own.litrown E Transformation job error
java.lang.IllegalStateException: Failed to add the track to the muxer
at android.media.MediaMuxer.nativeAddTrack(Native Method)
at android.media.MediaMuxer.addTrack(MediaMuxer.java:658)
at com.linkedin.android.litr.io.MediaMuxerMediaTarget.addTrack(MediaMuxerMediaTarget.java:116)
at com.linkedin.android.litr.transcoder.PassthroughTranscoder.processNextFrame(PassthroughTranscoder.java:79)
at com.linkedin.android.litr.TransformationJob.processNextFrame(TransformationJob.java:211)
at com.linkedin.android.litr.TransformationJob.transform(TransformationJob.java:108)
at com.linkedin.android.litr.TransformationJob.run(TransformationJob.java:77)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:463)
at java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1137)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:637)
at java.lang.Thread.run(Thread.java:1012)

[pawaom]

It works with AAC file but not an Mp3 file

[izzytwosheds]

Audio track encoding has to be compatible with H.264/AVC encoding. That is why AAC works and MP3 doesn't.
You can transcode to AAC by passing correct MediaFormat in setTargetFormat for audio track. There is a logic in MediaTransformer that enforces correct audio codec, you can use that as a starting point.

[pawaom]

Can you provide some sample how we do this.

I am trying to add overlay audio over video which previously contains some audio , It created a mp4 file with no video and the new audio removing old auido, can you guide me what needs to be changed, I am trying this code

Also this has been requested earlier by other developers, please provide documentation for various use cases, the demo files are good but better documentation like simple samples with just the uri to be used and mediaTransformer and various parameters it needs will be helpful, we have to go through demo code and try to modify it . For novice developers like me the learning curve gets a bit steep for some of the complex execution of this great library.

public class MainActivity extends AppCompatActivity {
 
    private File targetVideoFile;

    @Override
    protected void onCreate(@Nullable Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.frag2);
        Uri uri = Uri.parse("content://media/external/video/media/9368");

        TransformationListener tl = new TransformationListener() {

            @Override
            public void onStarted(String id) {
// TODO: Implement this method
                Toast.makeText(MainActivity.this, "Started ", Toast.LENGTH_SHORT).show();
            }

            @Override
            public void onProgress(String id, float progress) {
// TODO: Implement this method
                Toast.makeText(MainActivity.this, "progress " + progress, Toast.LENGTH_SHORT).show();
            }

            @Override
            public void onCompleted(String id, List trackTransformationInfos) {
                //sm("ok");
                Toast.makeText(MainActivity.this, "Completed ", Toast.LENGTH_SHORT).show();
            }

            @Override
            public void onCancelled(String id, List trackTransformationInfos) {
// TODO: Implement this method
            }

            @Override
            public void onError(String id, Throwable cause, List trackTransformationInfos) {
                // sm("error=" + id);
                Toast.makeText(MainActivity.this, cause.toString(), Toast.LENGTH_SHORT).show();
            }
        };
        Button TransFormButton = (Button) findViewById(R.id.TransformButton);
        TransFormButton.setOnClickListener(view -> {
            Toast.makeText(MainActivity.this, "clicked ", Toast.LENGTH_SHORT).show();
            TransformationOptions transformationOptions = new TransformationOptions.Builder()
                    .setGranularity(MediaTransformer.GRANULARITY_DEFAULT)
                    .setVideoFilters(null)
                    .build();
            File targetDirectory = getTargetFileDirectory();
            targetVideoFile = new File(targetDirectory, "transcoded_" + "abc.mp4");
            if (targetVideoFile.exists()) {
                Toast.makeText(MainActivity.this, "existed", Toast.LENGTH_SHORT).show();
                targetVideoFile.delete();
                Toast.makeText(MainActivity.this, "Deleted ", Toast.LENGTH_SHORT).show();
            }
            //---------SourceMedia----------------
            SourceMedia sourceMedia = new SourceMedia();
            setSourceMediaTracks(sourceMedia, uri);
            Uri audioUri = Uri.parse("content://media/external/audio/media/1000000008");
           // SourceMedia sourceMedia1 = new SourceMedia();
            setSourceMediaTracks(sourceMedia, audioUri);
            //---------TargetMedia----------------
            TargetMedia targetMedia = new TargetMedia();
            targetMedia.setTargetFile(targetVideoFile);
            targetMedia.setTracks(sourceMedia.tracks);

            MediaTarget mediaTarget = null;
            List<TrackTransform> trackTransforms = null;
            MediaSource mediaSource = null;
            try {
                mediaTarget = new MediaMuxerMediaTarget(
                        MainActivity.this,
                        Uri.fromFile(targetMedia.targetFile),
                        targetMedia.getIncludedTrackCount(),
                        0,
                        hasVp8OrVp9Track(targetMedia.tracks)
                                ? MediaMuxer.OutputFormat.MUXER_OUTPUT_WEBM
                                : MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
                trackTransforms = new ArrayList<>(targetMedia.tracks.size());
                MediaRange mediaRange = new MediaRange(0, Long.MAX_VALUE);
                mediaSource = new MediaExtractorMediaSource(MainActivity.this, sourceMedia.uri, mediaRange);
            } catch (MediaTargetException e) {
                Toast.makeText(MainActivity.this, "MediaTargetException  " + e.getMessage(), Toast.LENGTH_SHORT).show();
                throw new RuntimeException(e);
            } catch (MediaSourceException e) {
                Toast.makeText(MainActivity.this, "MediaSourceException  " + e.getMessage(), Toast.LENGTH_SHORT).show();
                throw new RuntimeException(e);
            }
            MediaTransformer mediaTransformer = new MediaTransformer(getApplicationContext());
            List<TargetTrack> tracks = targetMedia.tracks;
            for (int i = 0; i < tracks.size(); i++) {
                TargetTrack targetTrack = tracks.get(i);
                targetTrack.shouldInclude = true;
                if (!targetTrack.shouldInclude) {
                    continue;
                }
                Encoder encoder = new MediaCodecEncoder();
                TrackTransform.Builder trackTransformBuilder = new TrackTransform.Builder(mediaSource,
                        targetTrack.sourceTrackIndex,
                        mediaTarget)
                        .setTargetTrack(trackTransforms.size())
                        .setTargetFormat(targetTrack.shouldTranscode ? createMediaFormat(targetTrack) : null)
                        .setEncoder(encoder)
                        .setDecoder(new MediaCodecDecoder());
                if (targetTrack.format instanceof VideoTrackFormat) {
                    trackTransformBuilder.setRenderer(new GlVideoRenderer(createGlFilters(sourceMedia,
                            (TargetVideoTrack) targetTrack,
                            0.56f,
                            new PointF(0.6f, 0.4f),
                            30)));
                } else if (targetTrack.format instanceof AudioTrackFormat && targetTrack.overlay != null) {
                    trackTransformBuilder.setRenderer(
                            new AudioRenderer(
                                    encoder,
                                    Collections.singletonList(new AudioOverlayFilter(MainActivity.this, targetTrack.overlay))
                            )
                    );
                }
                trackTransforms.add(trackTransformBuilder.build());
            }
            mediaTransformer.transform("663600",
                    trackTransforms,
                    tl,
                    MediaTransformer.GRANULARITY_DEFAULT);
        });
    }

    private void setSourceMediaTracks(SourceMedia sourceMedia, Uri uri) {
        sourceMedia.uri = uri;
        sourceMedia.size = TranscoderUtils.getSize(MainActivity.this, uri);
        sourceMedia.duration = getMediaDuration(uri) / 1000f;
        try {
            MediaExtractor mediaExtractor = new MediaExtractor();
            mediaExtractor.setDataSource(MainActivity.this, uri, null);
            sourceMedia.tracks = new ArrayList<>(mediaExtractor.getTrackCount());
            for (int track = 0; track < mediaExtractor.getTrackCount(); track++) {
                MediaFormat mediaFormat = mediaExtractor.getTrackFormat(track);
                String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
                if (mimeType == null) {
                    continue;
                }
                if (mimeType.startsWith("video")) {
                    VideoTrackFormat videoTrack = new VideoTrackFormat(track, mimeType);
                    videoTrack.width = getInt(mediaFormat, MediaFormat.KEY_WIDTH);
                    videoTrack.height = getInt(mediaFormat, MediaFormat.KEY_HEIGHT);
                    videoTrack.duration = getLong(mediaFormat, MediaFormat.KEY_DURATION);
                    videoTrack.frameRate = MediaFormatUtils.getFrameRate(mediaFormat, -1).intValue();
                    videoTrack.keyFrameInterval = MediaFormatUtils.getIFrameInterval(mediaFormat, -1).intValue();
                    videoTrack.rotation = getInt(mediaFormat, KEY_ROTATION, 0);
                    videoTrack.bitrate = getInt(mediaFormat, MediaFormat.KEY_BIT_RATE);
                    sourceMedia.tracks.add(videoTrack);
                } else if (mimeType.startsWith("audio")) {
                    AudioTrackFormat audioTrack = new AudioTrackFormat(track, mimeType);
                    audioTrack.channelCount = getInt(mediaFormat, MediaFormat.KEY_CHANNEL_COUNT);
                    audioTrack.samplingRate = getInt(mediaFormat, MediaFormat.KEY_SAMPLE_RATE);
                    audioTrack.duration = getLong(mediaFormat, MediaFormat.KEY_DURATION);
                    audioTrack.bitrate = getInt(mediaFormat, MediaFormat.KEY_BIT_RATE);
                    sourceMedia.tracks.add(audioTrack);
                } else {
                    sourceMedia.tracks.add(new GenericTrackFormat(track, mimeType));
                }
            }
        } catch (IOException ex) {
            Log.e(TAG, "Failed to extract sourceMedia", ex);
        }
        sourceMedia.notifyChange();
    }

    @NonNull
    private File getTargetFileDirectory() {
        File targetDirectory = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES)
                + File.separator
                + "LiTr");
        if (!targetDirectory.exists()) {
            targetDirectory.mkdirs();
        }
        return targetDirectory;
    }

    private long getMediaDuration(@NonNull Uri uri) {
        MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
        mediaMetadataRetriever.setDataSource(MainActivity.this, uri);
        String durationStr = mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION);
        return Long.parseLong(durationStr);
    }

    private int getInt(@NonNull MediaFormat mediaFormat, @NonNull String key) {
        return getInt(mediaFormat, key, -1);
    }

    private int getInt(@NonNull MediaFormat mediaFormat, @NonNull String key, int defaultValue) {
        if (mediaFormat.containsKey(key)) {
            return mediaFormat.getInteger(key);
        }
        return defaultValue;
    }

    private long getLong(@NonNull MediaFormat mediaFormat, @NonNull String key) {
        if (mediaFormat.containsKey(key)) {
            return mediaFormat.getLong(key);
        }
        return -1;
    }

    private boolean hasVp8OrVp9Track(@NonNull List<TargetTrack> targetTracks) {
        for (TargetTrack targetTrack : targetTracks) {
            if (!targetTrack.shouldInclude) {
                continue;
            }
            if (TextUtils.equals(targetTrack.format.mimeType, MimeType.VIDEO_VP8)
                    || TextUtils.equals(targetTrack.format.mimeType, MimeType.VIDEO_VP9)) {
                return true;
            }
        }
        return false;
    }

    @Nullable
    private List<GlFilter> createGlFilters(@NonNull SourceMedia sourceMedia,
                                           @Nullable TargetVideoTrack targetTrack,
                                           float overlayWidth,
                                           @NonNull PointF position,
                                           float rotation) {
        List<GlFilter> glFilters = null;
        if (targetTrack != null && targetTrack.overlay != null) {
            try {
                Bitmap bitmap = BitmapFactory.decodeStream(MainActivity.this.getContentResolver().openInputStream(targetTrack.overlay));
                if (bitmap != null) {
                    float overlayHeight;
                    VideoTrackFormat sourceVideoTrackFormat = (VideoTrackFormat) sourceMedia.tracks.get(targetTrack.sourceTrackIndex);
                    if (sourceVideoTrackFormat.rotation == 90 || sourceVideoTrackFormat.rotation == 270) {
                        float overlayWidthPixels = overlayWidth * sourceVideoTrackFormat.height;
                        float overlayHeightPixels = overlayWidthPixels * bitmap.getHeight() / bitmap.getWidth();
                        overlayHeight = overlayHeightPixels / sourceVideoTrackFormat.width;
                    } else {
                        float overlayWidthPixels = overlayWidth * sourceVideoTrackFormat.width;
                        float overlayHeightPixels = overlayWidthPixels * bitmap.getHeight() / bitmap.getWidth();
                        overlayHeight = overlayHeightPixels / sourceVideoTrackFormat.height;
                    }
                    PointF size = new PointF(overlayWidth, overlayHeight);
                    GlFilter filter = TransformationUtil.createGlFilter(MainActivity.this,
                            targetTrack.overlay,
                            size,
                            position,
                            rotation);
                    if (filter != null) {
                        glFilters = new ArrayList<>();
                        glFilters.add(filter);
                    }
                }
            } catch (IOException ex) {
                Log.e(TAG, "Failed to extract audio track metadata: " + ex);
            }
        }
        return glFilters;
    }

    @Nullable
    private MediaFormat createMediaFormat(@Nullable TargetTrack targetTrack) {
        MediaFormat mediaFormat = null;
        if (targetTrack != null && targetTrack.format != null) {
            if (targetTrack.format.mimeType.startsWith("video")) {
                mediaFormat = createVideoMediaFormat((VideoTrackFormat) targetTrack.format);
            } else if (targetTrack.format.mimeType.startsWith("audio")) {
                mediaFormat = createAudioMediaFormat((AudioTrackFormat) targetTrack.format);
            }
        }
        return mediaFormat;
    }

    @NonNull
    private MediaFormat createVideoMediaFormat(@NonNull VideoTrackFormat trackFormat) {
        MediaFormat mediaFormat = new MediaFormat();
        mediaFormat.setString(MediaFormat.KEY_MIME, trackFormat.mimeType);
        mediaFormat.setInteger(MediaFormat.KEY_WIDTH, trackFormat.width);
        mediaFormat.setInteger(MediaFormat.KEY_HEIGHT, trackFormat.height);
        mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, trackFormat.bitrate);
        mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, trackFormat.keyFrameInterval);
        mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, trackFormat.frameRate);
        mediaFormat.setLong(MediaFormat.KEY_DURATION, trackFormat.duration);
        mediaFormat.setInteger(KEY_ROTATION, trackFormat.rotation);
        return mediaFormat;
    }

    @NonNull
    private MediaFormat createAudioMediaFormat(@NonNull AudioTrackFormat trackFormat) {
        MediaFormat mediaFormat = new MediaFormat();
        mediaFormat.setString(MediaFormat.KEY_MIME, trackFormat.mimeType);
        mediaFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, trackFormat.channelCount);
        mediaFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, trackFormat.samplingRate);
        mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, trackFormat.bitrate);
        mediaFormat.setLong(MediaFormat.KEY_DURATION, trackFormat.duration);
        return mediaFormat;
    }
}