So I create MediaToolAdapter and override onAddStream to change the codec.
public class SetVideoTranscodeListener extends MediaToolAdapter { @Override public void onAddStream(IAddStreamEvent event) { int streamIndex = event.getStreamIndex(); IStreamCoder streamCoder = event.getSource().getContainer().getStream(streamIndex).getStreamCoder(); if (streamCoder.getCodecType() == ICodec.Type.CODEC_TYPE_VIDEO) { streamCoder.setCodec(ICodec.findEncodingCodec(ICodec.ID.CODEC_ID_H264)); int retval = Configuration.configure("/usr/local/xuggler/share/ffmpeg/libx264-superfast.ffpreset", streamCoder); if (retval<0) throw new RuntimeException("cound not cofigure coder from preset file"); } super.onAddStream(event); } }And add an instance of this class as a listener to writer
writer.addListener(new SetVideoTranscodeListener());