Textura de superfície para textura de unidade
Eu vim com perguntas semelhantes antes, mas elas não eram bem esclarecidas e agora eu gostaria de seguir um conselho sobre o que estou fazendo de errado no meu código.
Então, o que estou tentando fazer é renderizar o SurfaceTexture do plug-in Android para o Unity Texture2D.
Aqui está o meu código Unity:
public class AndroidHandler : MonoBehaviour {
[SerializeField]
private RawImage _rawImage;
private Texture2D _inputTexture;
private AndroidJavaObject androidStreamerObj;
private System.IntPtr _nativePtr;
void Start () {
_rawImage.material.SetTextureScale("_MainTex", new Vector2(-1, -1));
InitAndroidStreamerObject();
}
private void InitAndroidStreamerObject()
{
androidStreamerObj = new AndroidJavaObject("makeitbetter.figazzz.com.vitamiousing7.AndroidStreamer");
Int32 texPtr = androidStreamerObj.Call <Int32> ("GetTexturePtr");
Debug.Log("texture pointer? " + texPtr);
Texture2D nativeTexture = Texture2D.CreateExternalTexture (128, 128, TextureFormat.RGBA32 , false, false, new System.IntPtr(texPtr));
_rawImage.texture = nativeTexture;
}
public void StartStream()
{
string streamLink = "rtmp://live.hkstv.hk.lxdns.com/live/hks"; //"rtsp://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mov"; //"rtmp://live.hkstv.hk.lxdns.com/live/hks";
androidStreamerObj.Call("LaunchStream", streamLink);
}
void Update()
{
androidStreamerObj.Call("DrawFrame");
}
}
Como você pode ver, é bem básico. Estou pedindo ao meu plugin do Android para criar o openGLTexture e estou usando o ponteiro da nova textura para alocar o Texture2D no Unity.
Em seguida, meu código de plug-in do Android:
public class AndroidStreamer {
private final int FLOAT_SIZE_BYTES = 4;
private final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
private final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
private Activity _currActivity;
private VideoView _streamConnection;
private Surface _cachedSurface;
private SurfaceTexture _cachedSurfaceTexture;
private Boolean isNewFrame = false;
//open gl
private int texWidth = 128;
private int texHeight = 128;
private float[] mMVPMatrix = new float[16];
private float[] mSTMatrix = new float[16];
private int glProgram;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;
private int unityTextureID = -1;
private int mTextureId = -1; //surface texture id
private int idFBO = -1;
private int idRBO = -1;
private final float[] mTriangleVerticesData = {
// X, Y, Z, U, V
-1.0f, -1.0f, 0, 0.f, 0.f,
1.0f, -1.0f, 0, 1.f, 0.f,
-1.0f, 1.0f, 0, 0.f, 1.f,
1.0f, 1.0f, 0, 1.f, 1.f,
};
private FloatBuffer mTriangleVertices;
private final String vertexShaderCode =
"uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uSTMatrix;\n" +
"attribute vec4 aPosition;\n" +
"attribute vec4 aTextureCoord;\n" +
"varying vec2 vTextureCoord;\n" +
"void main() {\n" +
" gl_Position = uMVPMatrix * aPosition;\n" +
" vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" +
"}\n";
private final String fragmentShaderCode =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" + // highp here doesn't seem to matter
"varying vec2 vTextureCoord;\n" +
"uniform samplerExternalOES sTexture;\n" +
"void main() {\n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"}\n";
public AndroidStreamer() {
Log.d("Unity", "AndroidStreamer was initialized");
_currActivity = UnityPlayer.currentActivity;
Vitamio.isInitialized(_currActivity);
_currActivity.runOnUiThread(new Runnable() {
@Override
public void run() {
_streamConnection = new VideoView(_currActivity);
_currActivity.addContentView(_streamConnection, new FrameLayout.LayoutParams(100, 100));
}
});
mTriangleVertices = ByteBuffer.allocateDirect(
mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
.order(ByteOrder.nativeOrder()).asFloatBuffer();
mTriangleVertices.put(mTriangleVerticesData).position(0);
Matrix.setIdentityM(mSTMatrix, 0);
initShaderProgram();
}
private void initShaderProgram()
{
Log.d("Unity", "initShaderProgram");
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexShaderCode);
int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderCode);
glProgram = GLES20.glCreateProgram();
GLES20.glAttachShader(glProgram, vertexShader);
checkGlError("glAttachVertexShader");
GLES20.glAttachShader(glProgram, fragmentShader);
checkGlError("glAttachFragmentShader");
GLES20.glLinkProgram(glProgram);
checkGlError("glLinkProgram");
maPositionHandle = GLES20.glGetAttribLocation(glProgram, "aPosition");
checkLocation(maPositionHandle, "aPosition");
maTextureHandle = GLES20.glGetAttribLocation(glProgram, "aTextureCoord");
checkLocation(maTextureHandle, "aTextureCoord");
muMVPMatrixHandle = GLES20.glGetUniformLocation(glProgram, "uMVPMatrix");
checkLocation(muMVPMatrixHandle, "uVMPMatrix");
muSTMatrixHandle = GLES20.glGetUniformLocation(glProgram, "uSTMatrix");
checkLocation(muSTMatrixHandle, "uSTMatrix");
}
private int loadShader(int shaderType, String source) {
int shader = GLES20.glCreateShader(shaderType);
if (shader != 0) {
GLES20.glShaderSource(shader, source);
GLES20.glCompileShader(shader);
int[] compiled = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
if (compiled[0] == 0) {
Log.e("Unity", "Could not compile shader " + shaderType + ":");
Log.e("Unity", GLES20.glGetShaderInfoLog(shader));
GLES20.glDeleteShader(shader);
shader = 0;
}
}
return shader;
}
private void checkLocation(int location, String label) {
if (location < 0) {
throw new RuntimeException("Unable to locate '" + label + "' in program");
}
}
private void checkGlError(String op) {
int error;
while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
Log.e("Unity", op + ": glError " + error);
throw new RuntimeException(op + ": glError " + error);
}
}
private void checkFrameBufferStatus()
{
int status = GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER);
checkGlError("glCheckFramebufferStatus");
switch (status)
{
case GLES20.GL_FRAMEBUFFER_COMPLETE:
Log.d("Unity", "complete");
break;
case GLES20.GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT:
Log.e("Unity", "incomplete attachment");
break;
case GLES20.GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT:
Log.e("Unity", "incomplete missing attachment");
break;
case GLES20.GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS:
Log.e("Unity", "incomplete dimensions");
break;
case GLES20.GL_FRAMEBUFFER_UNSUPPORTED:
Log.e("Unity", "framebuffer unsupported");
break;
default : Log.d("Unity", "default");
}
}
private void initGLTexture()
{
Log.d("Unity", "initGLTexture");
int textures[] = new int[1];
GLES20.glGenTextures(1, textures, 0);
checkGlError("glGenTextures initGLTexture");
mTextureId = textures[0];
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
checkGlError("glActiveTexture initGLTexture");
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureId);
checkGlError("glBindTexture initGLTexture");
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
checkGlError("glTexParameterf initGLTexture");
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
checkGlError("glTexParameterf initGLTexture");
}
public int GetTexturePtr()
{
Bitmap bitmap = Bitmap.createBitmap(texWidth, texHeight, Bitmap.Config.ARGB_8888);
for(int x = 0; x < texWidth; x++)
{
for (int y = 0; y < texHeight; y++)
{
bitmap.setPixel(x, y, Color.argb(155, 255, 50, 255));
}
}
Log.d("Unity", "Bitmap is: " + bitmap);
ByteBuffer buffer = ByteBuffer.allocate(bitmap.getByteCount());
bitmap.copyPixelsToBuffer(buffer);
//GLES20.glEnable(GLES11Ext.GL_TEXTURE_EXTERNAL_OES);
//checkGlError("glEnable GetTexturePtr");
int textures[] = new int[1];
GLES20.glGenTextures(1, textures, 0);
checkGlError("0");
unityTextureID = textures[0];
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
checkGlError("1");
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, unityTextureID);
checkGlError("2");
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, texWidth, texHeight, 0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null);
checkGlError("12");
//GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, bitmap, 0);
//checkGlError("3");
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
checkGlError("4");
GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
checkGlError("5");
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
checkGlError("6");
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
checkGlError("7");
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
checkGlError("8");
setupBuffers();
Log.d("Unity", "texture id returned: " + unityTextureID);
return unityTextureID;
}
private void setupBuffers()
{
Log.d("Unity", "setupBuffers");
//framebuffer
int buffers[] = new int[1];
GLES20.glGenFramebuffers(1, buffers, 0);
checkGlError("9");
idFBO = buffers[0];
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, idFBO);
checkGlError("10");
//render buffer
int rbuffers[] = new int[1];
GLES20.glGenRenderbuffers(1, rbuffers, 0);
checkGlError("glGenRenderBuffers setupBuffers");
idRBO = rbuffers[0];
GLES20.glBindRenderbuffer(GLES20.GL_RENDERBUFFER, idRBO);
checkGlError("glBindRenderBuffer setupBuffers");
GLES20.glRenderbufferStorage(GLES20.GL_RENDERBUFFER, GLES20.GL_RGBA4, texWidth, texHeight);
checkGlError("glRenderBufferStorage setupBuffers");
GLES20.glFramebufferRenderbuffer(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_RENDERBUFFER, idRBO);
checkGlError("glFramebufferRenderbuffer setupBuffers");
GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0, GLES20.GL_TEXTURE_2D, unityTextureID, 0);
checkGlError("glFrameBufferTexture2D");
checkFrameBufferStatus();
GLES20.glClearColor(1.0f, 0.5f, 0.0f, 1.0f);
checkGlError("glClearColor setupBuffers");
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
checkGlError("glClear setupBuffers");
}
public void DrawFrame()
{
if(isNewFrame && mSTMatrix != null) {
int[] testBuffer = new int[1];
GLES20.glGetIntegerv(GLES20.GL_FRAMEBUFFER_BINDING, testBuffer, 0);
Log.d("Unity", "DrawFrame binded = " + testBuffer[0] + " idFBO = " + idFBO);
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, idFBO);
checkGlError("glBindFrameBuffer DrawFrame");
GLES20.glClearColor(0.0f, 1.0f, 0.2f, 1.0f);
checkGlError("glClearColor DrawFrame");
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
checkGlError("glClear DrawFrame");
GLES20.glUseProgram(glProgram);
checkGlError("glUseProgram DrawFrame");
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
checkGlError("glActiveTexture DrawFrame");
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureId);
checkGlError("glBindTexture DrawFrame");
mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
GLES20.glVertexAttribPointer(maTextureHandle, 2, GLES20.GL_FLOAT, false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
checkGlError("glVertexAttribPointer DrawFrame");
GLES20.glEnableVertexAttribArray(maTextureHandle);
checkGlError("glEnableVertexAttribArray DrawFrame");
Matrix.setIdentityM(mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
checkGlError("glUniformMatrix4fv MVP onFrameAvailable");
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);
checkGlError("glUniformMatrix4fv ST onFrameAvailable");
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
checkGlError("glDrawArrays onFrameAvailable");
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
checkGlError("glBindFrameBuffer 0 onFrameAvailable");
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
checkGlError("glBindTexture onFrameAvailable");
isNewFrame = false;
}
}
public void LaunchStream(String streamLink) {
final String path = streamLink; //"http://dlqncdn.miaopai.com/stream/MVaux41A4lkuWloBbGUGaQ__.mp4"; //"rtmp://live.hkstv.hk.lxdns.com/live/hks";
Log.i("Unity", "hop hop1 = " + path);
_currActivity.runOnUiThread(new Runnable() {
@Override
public void run() {
_streamConnection.setVideoPath(path);
_streamConnection.setMediaController(new MediaController(_currActivity));
_streamConnection.requestFocus();
_streamConnection.setOnErrorListener(new MediaPlayer.OnErrorListener() {
@Override
public boolean onError(MediaPlayer mp, int what, int extra) {
Log.i("Unity", "some error, I don't know. what = " + what + " extra = " + extra);
return false;
}
});
_streamConnection.setOnPreparedListener(new MediaPlayer.OnPreparedListener() {
@Override
public void onPrepared(MediaPlayer mediaPlayer) {
// optional need Vitamio 4.0
Log.i("Unity", "hop hop5");
mediaPlayer.setPlaybackSpeed(1.0f);
}
});
initGLTexture();
_cachedSurfaceTexture = new SurfaceTexture(mTextureId);
_cachedSurfaceTexture.setDefaultBufferSize(texWidth, texHeight);
_cachedSurfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailableListener() {
@Override
public void onFrameAvailable(SurfaceTexture surfaceTexture) {
synchronized (this) {
surfaceTexture.updateTexImage();
mSTMatrix = new float[16];
surfaceTexture.getTransformMatrix(mSTMatrix);
isNewFrame = true;
}
}
});
_cachedSurface = new Surface(_cachedSurfaceTexture);
_streamConnection.setSurfaceToPlayer(_cachedSurface);
Log.i("Unity", "You're the best around!");
}
});
}
}
Decidi fornecer todo o código do meu plug-in Android para fornecer uma compreensão mais clara da situação que estou tendo. Basicamente, o que estou tentando fazer: 1) Chamo método "GetTexturePtr"do lado da Unity, criaGL_TEXTURE_2D textura que aplico ao Unity Texture2D. Também no lado Android, configuro o quadro e renderizo os buffers para alterar a cor dessa textura. Funciona bem porque preenche perfeitamente a cor. 2) Então chamo o método "LaunchStream", onde criaGL_TEXTURE_EXTERNAL_OES textura (em "initGLTexture()") e essa textura se aplica ao SurfaceTexture. 3) Também no UnityUpdate() método que eu chamo de método android "DrawFrame()", que deve atualizar minha textura do Unity de acordo com as alterações do SurfaceTexture.
No momento, estou usando o glError 1282GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, mTextureId); e é claro que a textura apenas preenche a cor verde
GLES20.glClearColor(0.0f, 1.0f, 0.2f, 1.0f);
checkGlError("glClearColor DrawFrame");
O que estou fazendo de errado?