抖音renderD128系统级疑难OOM分析与解决

复现机型：华为畅享10e(Android 10)

DRM

DRM是Linux内核层的显示驱动框架，它把显示功能封装成 open/close/ioctl 等标准接口，用户空间的程序调用这些接口，驱动设备，显示数据。libdrm库封装了DRM driver提供的这些接口。通过libdrm库，程序可以间接调用DRM Driver

#if defined(ANDROID) && !defined(__LP64__)extern void *__mmap2(void *, size_t, int, int, int, size_t);
static inline void *drm_mmap(void *addr, size_t length, int prot, int flags,                             int fd, loff_t offset){   /* offset must be aligned to 4096 (not necessarily the page size) */   if (offset & 4095) {      errno = EINVAL;      return MAP_FAILED;   }
   return __mmap2(addr, length, prot, flags, fd, (size_t) (offset >> 12));}#else/* assume large file support exists */#  define drm_mmap(addr, length, prot, flags, fd, offset) \              mmap(addr, length, prot, flags, fd, offset)

mesa3D

mesa3D中是通过调用libdrm库中的接口，间接调用DRM Driver的

https://gitlab.freedesktop.org/mesa/mesa

static int pvr_srv_bridge_call(int fd,                               uint8_t bridge_id,                               uint32_t function_id,                               void *input,                               uint32_t input_buffer_size,                               void *output,                               uint32_t output_buffer_size){   struct drm_srvkm_cmd cmd = {      .bridge_id = bridge_id,      .bridge_func_id = function_id,      .in_data_ptr = (uint64_t)(uintptr_t)input,      .out_data_ptr = (uint64_t)(uintptr_t)output,      .in_data_size = input_buffer_size,      .out_data_size = output_buffer_size,   };
   int ret = drmIoctl(fd, DRM_IOCTL_SRVKM_CMD, &cmd);   if (unlikely(ret))      return ret;
   VG(VALGRIND_MAKE_MEM_DEFINED(output, output_buffer_size));
   return 0U;}

#01 pc 00009f41  /data/app/com.example.crash.test-bqPIslSQVErr7gyFpcHl_w==/lib/arm/libnpth_vm_monitor.so （proxy_munmap)#02 pc 0001474b  /vendor/lib/libsrv_um.so#03 pc 000115d9  /vendor/lib/libsrv_um.so#04 pc 0000f061  /vendor/lib/libsrv_um.so（DevmemReleaseCpuVirtAddr+44）#05 pc 00015db1  /vendor/lib/egl/libGLESv2_mtk.so (glUnmapBuffer+536)#06 pc 003b865d  /system/lib/libhwui.so!libhwui.so (offset 0x244000) (GrGLBuffer::onUnmap()+54)#07 pc 001a0eb3  /system/lib/libhwui.so (GrResourceProvider::createPatternedIndexBuffer(unsigned short const*, int, int, int, GrUniqueKey const*)+174)#08 pc 001666b9  /system/lib/libhwui.so (GrResourceProvider::createQuadIndexBuffer()+24)#09 pc 00153df1  /system/lib/libhwui.so (GrResourceProvider::refQuadIndexBuffer()+44)#10 pc 001535c9  /system/lib/libhwui.so (GrAtlasTextOp::onPrepareDraws(GrMeshDrawOp::Target*)+328)

    #01 pc 0000edc1  /data/app/com.example.crash.test-o-BAwGot5UWCmlHJALMy2g==/lib/arm/libnpth_vm_monitor.so    #02 pc 0001d29b  /vendor/lib/libIMGegl.so    #03 pc 0001af31  /vendor/lib/libIMGegl.so    #04 pc 000187c1  /vendor/lib/libIMGegl.so    #05 pc 0001948b  /vendor/lib/libIMGegl.so    #06 pc 00018753  /vendor/lib/libIMGegl.so    #07 pc 0000b179  /vendor/lib/libIMGegl.so    #08 pc 0000f473  /vendor/lib/libIMGegl.so (IMGeglDestroySurface+462)    #09 pc 000171bd  /system/lib/libEGL.so (android::eglDestroySurfaceImpl(void*, void*)+48)    #10 pc 0025d40b  /system/lib/libhwui.so!libhwui.so (offset 0x245000) (android::uirenderer::renderthread::EglManager::destroySurface(void*)+30)    #11 pc 0025d2f7  /system/lib/libhwui.so!libhwui.so (offset 0x245000) (android::uirenderer::skiapipeline::SkiaOpenGLPipeline::setSurface(ANativeWindow*, android::uirenderer::renderthread::SwapBehavior, android::uirenderer::renderthrea     #12 pc 00244c03  /system/lib/libhwui.so!libhwui.so (offset 0x243000) (android::uirenderer::renderthread::CanvasContext::setSurface(android::sp<android::Surface>&&)+110)    #13 pc 00244af5  /system/lib/libhwui.so!libhwui.so (offset 0x243000) (android::uirenderer::renderthread::CanvasContext::destroy()+48)    #15 pc 0023015f  /system/lib/libhwui.so!libhwui.so (offset 0x208000) (std::__1::packaged_task<void ()>::operator()()+50)    #16 pc 0020da97  /system/lib/libhwui.so!libhwui.so (offset 0x208000) (android::uirenderer::WorkQueue::process()+158)    #17 pc 0020d8f5  /system/lib/libhwui.so!libhwui.so (offset 0x208000) (android::uirenderer::renderthread::RenderThread::threadLoop()+72)    #18 pc 0000d91b  /system/lib/libutils.so (android::Thread::_threadLoop(void*)+182)    #19 pc 0009b543  /apex/com.android.runtime/lib/bionic/libc.so!libc.so (offset 0x8d000) (__pthread_start(void*)+20)

static void release_freed_buffer(pb_ctx_t* ctx) {    /** 一些检查和判空操作会省略 **/    ...    /** 阈值检查 **/    if (!limit_check(ctx)) return;
    // 拿到buffer_list    pb_buffer_list_t* buffers_list = ctx->pb_global->buffers_list;
    pb_buffer_info_t *buffer_info, *prev_info;    for (int i = 0; i < 5; i++) {        buffer_info = buffer_info->buffers[i];        if (buffer_info == NULL) continue;
        /** 第一个buffer不释放，简化逻辑 **/        while(buffer_info) {            prev_info = buffer_info;            buffer_info = buffer_info->next;
            if (buffer_info && buffer_info->flag == 0) {                int ret = pvrsrvReleaseCPUMapping((void**)buffer_info->sparse_buffer->cpu_mapping_info->info);
                LOGE("%s, release cpu mapping ret: %d", __FUNCTION__, ret);                if (ret == 0) {                    buffer_info->flag = 1;                    buffer_info->sparse_buffer->mmap_ptr = NULL;                    prev_info->next = buffer_info->next;                    buffers_list->buffer_size[i]--;                    free(buffer_info);                    buffer_info = prev_info;                }            }        }    }}

测试环境和方式与前面“问题复现”章节一致

#define ORIGIN_TIME_LIMIT_INST      0x5230f247 // 30s#define NEW_TIME_LIMIT_INST      0x32e8f240 // 1s 提前构造好的指令编码#define FUNC_SYM "_ZN7android10uirenderer12renderthread12CacheManager18trimStaleResourcesEv"
static void change_destroy_wait_time() {    /** 一些检查和判空操作会省略 **/#ifdef __arm__    void* handle = dlopen("libhwui.so");    // 从trimStaleResources方法的起始地址开始搜索内存    void* sym_ptr = dlsym(handle, FUNC_SYM);
    sym_ptr = (void*)((uint32_t)sym_ptr & 0xfffffffc);
    uint32_t* inst_start = (uint32_t*)sym_ptr;    uint32_t* search_limit = inst_start + 12;
    while(inst_start < search_limit) {        /* 找到并修改对应指令 */        if (*inst_start == ORIGIN_TIME_LIMIT_INST) {            if(mprotect((void*)((uint32_t)inst_start & (unsigned int) PAGE_MASK), PAGE_SIZE, PROT_READ | PROT_WRITE | PROT_EXEC)) {                return;            }
            *inst_start = NEW_TIME_LIMIT_INST;            flash_page_cache(inst_start);
            if(mprotect((void*)((uint32_t)inst_start & (unsigned int) PAGE_MASK), PAGE_SIZE, PROT_READ|PROT_WRITE|PROT_EXEC)) {                return;            }            break;        }
        inst_start++;    }#endif}

KEGLGetPoolBuffers end, a1: 0xd1b95068, a2: 0x1, ret: 0x1 ==> before: 1265852416, after: 1292066816, alloc: 26214400KEGLGetPoolBuffers end, a1: 0xd1b95068, a2: 0x1, ret: 0x1 ==> before: 1292066816, after: 1318281216, alloc: 26214400KEGLGetPoolBuffers end, a1: 0xd1b95068, a2: 0x1, ret: 0x0 ==> before: 1318281216, after: 1318281216, alloc: 0KEGLReleasePoolBuffers end, ret: 1 ==> before: 1318281216, after: 1318281216, release: 0KEGLReleasePoolBuffers end, ret: 1 ==> before: 1318281216, after: 1318281216, release: 0KEGLReleasePoolBuffers end, ret: 1 ==> before: 1318281216, after: 1318281216, release: 0...KEGLReleasePoolBuffers end, ret: 1 ==> before: 1318281216, after: 1318281216, release: 0KEGLReleasePoolBuffers end, ret: 1 ==> before: 1318281216, after: 1318281216, release: 0KEGLReleasePoolBuffers end, ret: 1 ==> before: 1318281216, after: 1318281216, release: 0KEGLGetPoolBuffers end, a1: 0xd1b95068, a2: 0x1, ret: 0x1 ==> before: 1318281216, after: 1318281216, alloc: 0KEGLGetPoolBuffers end, a1: 0xd1b95068, a2: 0x1, ret: 0x1 ==> before: 1318281216, after: 1318281216, alloc: 0KEGLGetPoolBuffers end, a1: 0xd1b95068, a2: 0x1, ret: 0x1 ==> before: 1318281216, after: 1318281216, alloc: 0

#01 pc 0000ebf5  /data/app/com.example.crash.test-1hHKnp6FBSv-HjrVtXQo1Q==/lib/arm/libnpth_vm_monitor.so （proxy_KEGLReleasePoolBuffers)#02 pc 00047c2d  /vendor/lib/egl/libGLESv2_mtk.so#03 pc 00046a7b  /vendor/lib/egl/libGLESv2_mtk.so (ResetSurface)#04 pc 00028bf7  /vendor/lib/egl/libGLESv2_mtk.so#05 pc 000d2165  /vendor/lib/egl/libGLESv2_mtk.so (RM_FlushHWQueue)#06 pc 00028c73  /vendor/lib/egl/libGLESv2_mtk.so #07 pc 000453fd  /vendor/lib/egl/libGLESv2_mtk.so (PrepareToDraw)#08 pc 0001d977  /vendor/lib/egl/libGLESv2_mtk.so (glDrawArrays+738)#09 pc 00009edd  /system/lib/libGameGraphicsOpt.so (hw_glDrawArraysHookV2+18)#10 pc 001d1769  /system/lib/libhwui.so (GrGLGpu::sendMeshToGpu(GrPrimitiveType, GrBuffer const*, int, int)+74)#11 pc 001d15f3  /system/lib/libhwui.so (GrMesh::sendToGpu(GrMesh::SendToGpuImpl*) const+38)#12 pc 001d13e5  /system/lib/libhwui.so (GrGLGpu::draw(GrRenderTarget*, GrSurfaceOrigin, GrPrimitiveProcessor const&, GrPipeline const

int opt_mtk_buffer(int api_level, int new_buffer_size) {    ...(无关代码省略）    if (check_buffer_size(new_buffer_size)) {        prefered_buffer_size = new_buffer_size;    }
    KEGLGetPoolBuffers_stub = bytehook_hook_single(            "libGLESv2_mtk.so",            NULL,            "KEGLGetPoolBuffers",            (void*)proxy_KEGLGetPoolBuffers,            (bytehook_hooked_t)bytehook_hooked_mtk,            NULL);    ...(无关代码省略）
    return 0;}
static void* proxy_KEGLGetPoolBuffers(void** a1, void* a2, int a3, int a4) {    //修改buffer_limits    modify_buffer_size((pb_ctx_t*)a1);    void* ret = BYTEHOOK_CALL_PREV(proxy_KEGLGetPoolBuffers, KEGLGetPoolBuffers_t, a1, a2, a3, a4);    BYTEHOOK_POP_STACK();    return ret;}
static void modify_buffer_size(pb_ctx_t* ctx) {    if (__predict_false(ctx == NULL || ctx->node == NULL || ctx->node->buffer_inner == NULL)) {        return;    }
    if (ctx->node->buffer_inner->num == ORIGIN_BUFFER_SIZE) {        ctx->node->buffer_inner->num = prefered_buffer_size;    }}