Optimise Android rendering

This commit is contained in:
Niellun
2025-06-02 04:21:16 +03:00
parent 907247467d
commit cdc68e4191
13 changed files with 137 additions and 260 deletions
+13 -21
View File
@@ -140,36 +140,28 @@ bool Connector::nextState(u_int8_t state)
if (state == _state)
return false;
if (state > _state)
if (state == PROTOCOL_STATUS_ERROR && _failCount++ < 10)
{
_failCount++;
return false;
}
if (state > _state || state == PROTOCOL_STATUS_INITIALISING)
{
_nodeviceCount = 0;
_failCount = 0;
_state = state;
return true;
}
switch (state)
if (state == PROTOCOL_STATUS_NO_DEVICE && (_nodeviceCount++ > 10 || _state >= PROTOCOL_STATUS_ONLINE))
{
case PROTOCOL_STATUS_INITIALISING:
break;
case PROTOCOL_STATUS_ERROR:
_nodeviceCount = 0;
if (_failCount++ < 10)
return false;
break;
case PROTOCOL_STATUS_NO_DEVICE:
if (_nodeviceCount++ < 10 && _state < PROTOCOL_STATUS_ONLINE)
return false;
break;
default:
return false;
_failCount = 0;
_state = state;
return true;
}
_state = state;
return true;
return false;
}
void Connector::state(u_int8_t state)
+13 -7
View File
@@ -5,6 +5,7 @@
#include "settings.h"
Decoder::Decoder()
: _context(nullptr)
{
}
@@ -34,6 +35,12 @@ void Decoder::stop()
_thread.join();
}
void Decoder::flush()
{
if(_context)
avcodec_flush_buffers(_context);
}
// Initialize and select the best decoder (try HW first, then SW)
AVCodecContext *Decoder::load_codec(AVCodecID codec_id)
{
@@ -100,10 +107,10 @@ void Decoder::runner()
setThreadName("video-decoding");
// Load codec context
AVCodecContext *context = load_codec(_codecId);
if (_status.null(context, ("Can't find decoder for codec " + _codecId)))
_context = load_codec(_codecId);
if (_status.null(_context, ("Can't find decoder for codec " + _codecId)))
return;
std::string codec = context->codec->name;
std::string codec = _context->codec->name;
// Initialize parser for the codec
AVCodecParserContext *parser = av_parser_init(_codecId);
@@ -117,14 +124,15 @@ void Decoder::runner()
AVFrame *frame = av_frame_alloc();
if (!_status.null(frame, "Can't allocate frame for codec " + codec))
{
loop(context, parser, packet, frame); // Run decoding loop
loop(_context, parser, packet, frame); // Run decoding loop
av_frame_free(&frame);
}
av_packet_free(&packet);
}
av_parser_close(parser);
}
avcodec_free_context(&context);
avcodec_free_context(&_context);
_context = nullptr;
if (_status.error())
std::cout << "[Video] Decoder error: " << _status.message() << std::endl;
@@ -149,8 +157,6 @@ void Decoder::loop(AVCodecContext *context, AVCodecParserContext *parser, AVPack
uint8_t *paket_data;
int paket_size;
// printf("avparser offset %d size %d fullsize %d\n", data_ptr-segment.data, data_size, segment.size);
// Parse raw data into packets
int len = av_parser_parse2(parser, context,
&paket_data, &paket_size,
+2
View File
@@ -24,6 +24,7 @@ public:
void start(AtomicQueue<Message> *data, VideoBuffer *vb, AVCodecID codecId);
void stop();
void flush();
private:
void runner();
@@ -31,6 +32,7 @@ private:
static AVCodecContext *load_codec(AVCodecID codec_id);
std::thread _thread;
AVCodecContext* _context;
AVCodecID _codecId;
Error _status;
+2 -2
View File
@@ -3,8 +3,8 @@
#define PROTOCOL_STATUS_INITIALISING 0 // Initialised > 1
#define PROTOCOL_STATUS_NO_DEVICE 1 // Start linking > 3
#define PROTOCOL_STATUS_ERROR 2 // Linked > 4, no device in sequence > 1
#define PROTOCOL_STATUS_LINKING 3 // Linked > 4, Failed in sequence > 2
#define PROTOCOL_STATUS_LINKING 2 // Linked > 4, Failed in sequence > 3
#define PROTOCOL_STATUS_ERROR 3 // Linked > 4, no device in sequence > 1
#define PROTOCOL_STATUS_ONLINE 4 // Phone connected > 5, no device > 1
#define PROTOCOL_STATUS_CONNECTED 5 // Phone disconnected > 4, no device > 1
+6 -4
View File
@@ -36,16 +36,18 @@ bool Interface::drawHome(bool force, int state)
if (_textDongle.prepare(_renderer, "Connection error", colorError))
_textDongle.draw(_renderer, 0.05 * width, 0.2 * height - _textDongle.height / 2);
}
if (_textDongle.prepare(_renderer, "Insert dongle", state == PROTOCOL_STATUS_NO_DEVICE ? color1 : color1_inactive))
_textDongle.draw(_renderer, 0.05 * width, 0.2 * height - _textDongle.height / 2);
else
{
if (_textDongle.prepare(_renderer, "Insert dongle", state == PROTOCOL_STATUS_NO_DEVICE ? color1 : color1_inactive))
_textDongle.draw(_renderer, 0.05 * width, 0.2 * height - _textDongle.height / 2);
}
if (_textInit.prepare(_renderer, "Initialising", state == PROTOCOL_STATUS_LINKING ? color2 : color2_inactive))
_textInit.draw(_renderer, 0.05 * width, 0.4 * height - _textInit.height / 2);
if (_textConnect.prepare(_renderer, "Connect phone", state == PROTOCOL_STATUS_ONLINE ? color3 : color3_inactive))
_textConnect.draw(_renderer, 0.05 * width, 0.6 * height - _textConnect.height / 2);
if (_textLaunch.prepare(_renderer, "Launching", state == PROTOCOL_STATUS_CONNECTED? color4 : color4_inactive))
if (_textLaunch.prepare(_renderer, "Launching", state == PROTOCOL_STATUS_CONNECTED ? color4 : color4_inactive))
_textLaunch.draw(_renderer, 0.05 * width, 0.8 * height - _textLaunch.height / 2);
SDL_RenderPresent(_renderer);
return true;
}
+21 -16
View File
@@ -41,8 +41,6 @@ struct RunParams
uint8_t deviceStatus;
uint32_t frameDelay;
int activeDelay;
float cropX;
float cropY;
};
void processKey(Protocol &protocol, SDL_Keysym key, RunParams &params)
@@ -59,6 +57,10 @@ void processKey(Protocol &protocol, SDL_Keysym key, RunParams &params)
active = false;
break;
case SDLK_r:
params.dirty = true;
break;
case SDLK_LEFT:
protocol.sendKey(100);
break;
@@ -78,8 +80,9 @@ void processKey(Protocol &protocol, SDL_Keysym key, RunParams &params)
}
}
void processEvents(Protocol &protocol, RunParams &params, VideoBuffer &vb)
bool processEvents(Protocol &protocol, RunParams &params, Renderer &renderer)
{
bool result = false;
SDL_Event e;
int motionX = -1;
int motionY = -1;
@@ -139,11 +142,8 @@ void processEvents(Protocol &protocol, RunParams &params, VideoBuffer &vb)
params.dirty = true;
params.videoRendered = false;
params.frameDelay = params.connected ? params.activeDelay : FRAME_DELAY_INACTIVE;
if (!params.connected)
{
vb.reset();
params.videoPrepaired = false;
}
params.videoPrepaired = false;
result = true;
}
else if (e.type == evtStatus)
{
@@ -158,12 +158,14 @@ void processEvents(Protocol &protocol, RunParams &params, VideoBuffer &vb)
int window_width, window_height;
SDL_GetWindowSize(window, &window_width, &window_height);
if (downX >= 0)
protocol.sendClick(params.cropX * downX / window_width, params.cropY * downY / window_height, true);
protocol.sendClick(renderer.xScale * downX / window_width, renderer.yScale * downY / window_height, true);
if (motionX >= 0)
protocol.sendMove(params.cropX * motionX / window_width, params.cropY * motionY / window_height);
protocol.sendMove(renderer.xScale * motionX / window_width, renderer.yScale * motionY / window_height);
if (upX >= 0)
protocol.sendClick(params.cropX * upX / window_width, params.cropY * upY / window_height, false);
protocol.sendClick(renderer.xScale * upX / window_width, renderer.yScale * upY / window_height, false);
}
return result;
}
void application()
@@ -178,8 +180,6 @@ void application()
p.videoRendered = false;
p.fullscreen = Settings::fullscreen;
p.mouseDown = false;
p.cropX = 1;
p.cropY = 1;
if (p.fullscreen)
{
@@ -206,7 +206,14 @@ void application()
Uint32 frameStart = SDL_GetTicks();
while (active)
{
processEvents(protocol, p, videoBuffer);
if(processEvents(protocol, p, interface))
{
if(p.connected)
{
decoder.flush();
videoBuffer.reset();
}
}
if (p.connected)
{
@@ -214,8 +221,6 @@ void application()
uint32_t frameid = 0;
if (videoBuffer.latest(&frame, &frameid) && (frameid != latestid || p.dirty) && frame)
{
if (!p.videoPrepaired)
p.videoPrepaired = interface.prepare(frame, Settings::width, Settings::height, Settings::scaler, protocol.phoneAndroid, &p.cropX, &p.cropY);
if (interface.render(frame))
{
p.videoRendered = true;
+25 -63
View File
@@ -12,11 +12,9 @@ Protocol::Protocol(uint16_t width, uint16_t height, uint16_t fps, uint16_t paddi
audioStreamMain(Settings::audioQueue),
audioStreamAux(Settings::audioQueue),
phoneConnected(false),
phoneAndroid(false),
_width(width),
_height(height),
_fps(fps),
_phoneInfo(false)
_fps(fps)
{
}
@@ -65,19 +63,33 @@ void Protocol::sendConfig()
if (Settings::micType == 3)
mic = 21;
float aspect = (float)_width / _height;
int height = 480;
if (Settings::androidMode == 2)
height = 1280;
if (Settings::androidMode == 3)
height = 1920;
int width = aspect*height;
if (aspect < 1)
int width;
int height;
switch (Settings::androidMode)
{
width = height;
height = height/aspect;
default:
width = 800;
height = 480;
break;
case 2:
width = 1280;
height = 720;
break;
case 3:
width = 1920;
height = 1080;
break;
}
if (_width < _height)
std::swap(width, height);
float scale = std::min((float)width / _width, (float)height / _height);
width = _width * scale;
height = _height * scale;
std::cout << "[Protocol] Request android image " << width << "x" << height << std::endl;
char buffer[512];
snprintf(buffer, sizeof(buffer),
"{\"syncTime\":%d,\"mediaDelay\":%d,\"drivePosition\":%d,"
@@ -206,42 +218,6 @@ void Protocol::sendEncryption()
connector.send(CMD_ENCRYPTION, false, buf, 4);
}
bool Protocol::jsonFind(const char *json, uint16_t length, const char *key, char *value, uint16_t size) const
{
size_t key_len = std::strlen(key);
const char *end = json + length;
const char *p = json;
while (p < end - key_len - 3)
{
if (*p == '"' && *(p + 1 + key_len) == '"' && std::memcmp(p + 1, key, key_len) == 0)
{
const char *colon = p + 1 + key_len + 1;
while (colon < end && *colon == ' ')
++colon;
if (colon >= end || *colon != ':')
return false;
const char *val_start = colon + 1;
while (val_start < end && (*val_start == ' ' || *val_start == '"'))
++val_start;
const char *val_end = val_start;
while (val_end < end && *val_end != ',' && *val_end != '}' && *val_end != '"')
++val_end;
uint16_t val_len = val_end - val_start;
if (val_len + 1 > size)
return false;
std::memcpy(value, val_start, val_len);
value[val_len] = '\0';
return true;
}
++p;
}
return false;
}
void Protocol::onStatus(uint8_t status)
{
pushEvent(_evtStatusId, status);
@@ -288,26 +264,12 @@ void Protocol::onData(uint32_t cmd, uint32_t length, uint8_t *data)
bool dispose = true;
switch (cmd)
{
case CMD_JSON_CONTROL:
if (!_phoneInfo)
{
char res[32];
if (jsonFind((char *)data, length, "MDLinkType", res, 32))
{
_phoneInfo = true;
phoneAndroid = strncmp(res, "AndroidAuto", 11) == 0;
}
}
break;
case CMD_PLUGGED:
onPhone(true);
break;
case CMD_UNPLUGGED:
onPhone(false);
_phoneInfo = false;
phoneAndroid = false;
break;
case CMD_VIDEO_DATA:
-2
View File
@@ -34,7 +34,6 @@ public:
AtomicQueue<Message> audioStreamMain;
AtomicQueue<Message> audioStreamAux;
bool phoneConnected;
bool phoneAndroid;
private:
void sendInt(uint32_t cmd, uint32_t value, bool encryption = true);
@@ -54,7 +53,6 @@ private:
uint16_t _width;
uint16_t _height;
uint16_t _fps;
bool _phoneInfo;
uint32_t _evtStatusId = (uint32_t)-1;
uint32_t _evtPhoneId = (uint32_t)-1;
+50 -116
View File
@@ -141,20 +141,19 @@ SDL_Rect RendererImage::draw(SDL_Renderer *renderer, int w, int h)
}
const Renderer::FormatMapping Renderer::_mapping[] = {
{AV_PIX_FMT_RGB24, SDL_PIXELFORMAT_RGB24, &Renderer::rgb, &Renderer::crgb, "RGB24", 3},
{AV_PIX_FMT_YUV420P, SDL_PIXELFORMAT_IYUV, &Renderer::yuv, &Renderer::cyuv, "YUV420P", 0},
{AV_PIX_FMT_YUVJ420P, SDL_PIXELFORMAT_IYUV, &Renderer::yuv, &Renderer::cyuv, "YUVJ420P", 0},
{AV_PIX_FMT_NV12, SDL_PIXELFORMAT_NV12, &Renderer::nv, &Renderer::cnv, "NV12", 0}};
{AV_PIX_FMT_RGB24, SDL_PIXELFORMAT_RGB24, &Renderer::rgb, "RGB24"},
{AV_PIX_FMT_YUV420P, SDL_PIXELFORMAT_IYUV, &Renderer::yuv, "YUV420P"},
{AV_PIX_FMT_YUVJ420P, SDL_PIXELFORMAT_IYUV, &Renderer::yuv, "YUVJ420P"},
{AV_PIX_FMT_NV12, SDL_PIXELFORMAT_NV12, &Renderer::nv, "NV12"}};
Renderer::Renderer(SDL_Renderer *renderer)
: _renderer(renderer),
: xScale(0),
yScale(0),
_renderer(renderer),
_texture(nullptr),
_textureWidth(0),
_textureHeight(0),
_cropX(0),
_cropY(0),
_crop(false),
_bytesPerPixel(0),
_sourceRect({0, 0, 0, 0}),
_render(nullptr),
_sws(nullptr),
_frame(nullptr)
@@ -168,25 +167,21 @@ Renderer::~Renderer()
bool Renderer::render(AVFrame *frame)
{
if (_render == nullptr || _texture == nullptr)
return false;
if (_render == nullptr || frame->width != _textureWidth || frame->height != _textureHeight)
{
clear();
if (!prepare(frame, Settings::width, Settings::height))
return false;
}
(this->*_render)(frame);
SDL_RenderClear(_renderer);
SDL_RenderCopy(_renderer, _texture, nullptr, nullptr);
SDL_RenderCopy(_renderer, _texture, &_sourceRect, nullptr);
SDL_RenderPresent(_renderer);
return true;
}
bool Renderer::prepareTexture(uint32_t format, int width, int height)
{
if (_texture)
{
if (_textureWidth == width && _textureHeight == height)
return true;
SDL_DestroyTexture(_texture);
_texture = nullptr;
}
_texture = SDL_CreateTexture(_renderer, format,
SDL_TEXTUREACCESS_STREAMING,
width, height);
@@ -195,6 +190,9 @@ bool Renderer::prepareTexture(uint32_t format, int width, int height)
std::cerr << "[UX] SDL can't create video texture: " << SDL_GetError() << std::endl;
return false;
}
_textureWidth = width;
_textureHeight = height;
return true;
}
@@ -217,62 +215,44 @@ void Renderer::clear()
}
}
bool Renderer::prepare(AVFrame *frame, int targetWidth, int targetHeight, uint32_t scaler, bool android, float *cropX, float *cropY)
bool Renderer::prepare(AVFrame *frame, int targetWidth, int targetHeight)
{
clear();
std::cout << "[UX] Prepare renderer " << targetWidth << "x" << targetHeight << " for source " << frame->width << "x" << frame->height << " " << (android ? "android auto" : "carplay") << std::endl;
if (targetWidth == 0 || targetHeight == 0)
return false;
bool scaled = frame->height * targetWidth == targetHeight * frame->width;
int width = frame->width;
int height = frame->height;
float scale = (float)frame->width / targetWidth;
float scale2 = (float)frame->height / targetHeight;
if (scale > scale2)
scale = scale2;
int width = targetWidth * scale;
int height = targetHeight * scale;
if (android && !scaled)
_sourceRect = {(frame->width - width) / 2, (frame->height - height) / 2, width, height};
xScale = (float)width / frame->width;
yScale = (float)height / frame->height;
std::cout << "[UX] Prepare renderer " << width << "x" << height << " for source " << frame->width << "x" << frame->height << " target " << targetWidth << "x" << targetHeight << std::endl;
AVPixelFormat fmt = static_cast<AVPixelFormat>(frame->format);
for (const FormatMapping &mapping : _mapping)
{
float scale = (float)frame->width / targetWidth;
float scale2 = (float)frame->height / targetHeight;
if (scale > scale2)
scale = scale2;
width = targetWidth * scale;
height = targetHeight * scale;
}
bool cropW = android && width != frame->width;
bool cropH = android && height != frame->height;
_cropX = cropW ? (frame->width - width) / 2 : 0;
_cropY = cropH ? (frame->height - height) / 2 : 0;
*cropX = cropW ? (float)width / frame->width : 1;
*cropY = cropH ? (float)height / frame->height : 1;
if (scaled || cropW || cropH)
{
AVPixelFormat fmt = static_cast<AVPixelFormat>(frame->format);
for (const FormatMapping &mapping : _mapping)
if (mapping.avFormat == fmt)
{
if (mapping.avFormat == fmt)
if (prepareTexture(mapping.sdlFormat, frame->width, frame->height))
{
if (prepareTexture(mapping.sdlFormat, width, height))
{
std::cout << "[UX] Direct rendering " << mapping.name << std::endl;
_render = (cropW || cropH) ? mapping.functionCrop : mapping.function;
_bytesPerPixel = mapping.bpp;
return true;
}
std::cout << "[UX] Direct rendering " << mapping.name << std::endl;
_render = mapping.function;
return true;
}
}
}
else
{
std::cout << "[UX] Scaling required from " << frame->width << "x" << frame->height << " to " << targetWidth << "x" << targetHeight << std::endl;
}
if (!prepareTexture(SDL_PIXELFORMAT_IYUV, width, height))
if (!prepareTexture(SDL_PIXELFORMAT_IYUV, frame->width, frame->height))
return false;
_sws = sws_getContext(frame->width, frame->height, (AVPixelFormat)frame->format,
width, height, AV_PIX_FMT_YUV420P,
scaler, nullptr, nullptr, nullptr);
frame->width, frame->height, AV_PIX_FMT_YUV420P,
SWS_BILINEAR, nullptr, nullptr, nullptr);
if (!_sws)
{
std::cerr << "[UX] Can't create sws context" << std::endl;
@@ -286,8 +266,8 @@ bool Renderer::prepare(AVFrame *frame, int targetWidth, int targetHeight, uint32
return false;
}
_frame->format = AV_PIX_FMT_YUV420P;
_frame->width = width;
_frame->height = height;
_frame->width = frame->width;
_frame->height = frame->height;
// Allocate data buffer with 32 byte allingment
int avRes = av_frame_get_buffer(_frame, 32);
if (avRes != 0)
@@ -309,15 +289,6 @@ void Renderer::rgb(AVFrame *frame)
frame->data[0], frame->linesize[0]);
}
void Renderer::crgb(AVFrame *frame)
{
uint8_t *rgb_data = frame->data[0] + _cropY * frame->linesize[0] + _cropX * _bytesPerPixel;
SDL_UpdateTexture(
_texture,
nullptr,
rgb_data, frame->linesize[0]);
}
void Renderer::nv(AVFrame *frame)
{
SDL_UpdateNVTexture(
@@ -327,20 +298,6 @@ void Renderer::nv(AVFrame *frame)
frame->data[1], frame->linesize[1]);
}
void Renderer::cnv(AVFrame *frame)
{
uint8_t *y_plane = frame->data[0] + _cropY * frame->linesize[0] + _cropX;
// UV plane (subsampled by 2)
uint8_t *uv_plane = frame->data[1] + (_cropY / 2) * frame->linesize[1] + 2 * (_cropX / 2);
SDL_UpdateNVTexture(
_texture,
nullptr,
y_plane, frame->linesize[0],
uv_plane, frame->linesize[1]);
}
void Renderer::yuv(AVFrame *frame)
{
SDL_UpdateYUVTexture(
@@ -351,42 +308,19 @@ void Renderer::yuv(AVFrame *frame)
frame->data[2], frame->linesize[2]);
}
void Renderer::cyuv(AVFrame *frame)
{
int crop_x_chroma = _cropX / 2;
int crop_y_chroma = _cropY / 2;
uint8_t *y_plane = frame->data[0] + _cropY * frame->linesize[0] + _cropX;
uint8_t *u_plane = frame->data[1] + crop_y_chroma * frame->linesize[1] + crop_x_chroma;
uint8_t *v_plane = frame->data[2] + crop_y_chroma * frame->linesize[2] + crop_x_chroma;
SDL_UpdateYUVTexture(
_texture,
nullptr,
y_plane, frame->linesize[0],
u_plane, frame->linesize[1],
v_plane, frame->linesize[2]);
}
void Renderer::scale(AVFrame *frame)
{
// Scale frame to output format
sws_scale(_sws,
frame->data, frame->linesize,
0, _textureHeight,
0, _frame->height,
_frame->data,
_frame->linesize);
yuv(_frame);
SDL_UpdateYUVTexture(
_texture,
nullptr,
_frame->data[0], _frame->linesize[0],
_frame->data[1], _frame->linesize[1],
_frame->data[2], _frame->linesize[2]);
}
void Renderer::cscale(AVFrame *frame)
{
// Scale frame to output format
sws_scale(_sws,
frame->data, frame->linesize,
0, _textureHeight,
_frame->data,
_frame->linesize);
cyuv(_frame);
}
+4 -12
View File
@@ -51,8 +51,9 @@ public:
Renderer(SDL_Renderer *renderer);
~Renderer();
bool prepare(AVFrame *frame, int targetWidth, int targetHeight, uint32_t scaler, bool android, float *cropX, float *cropY);
bool render(AVFrame *frame);
float xScale;
float yScale;
protected:
SDL_Renderer *_renderer;
@@ -65,31 +66,22 @@ private:
AVPixelFormat avFormat;
SDL_PixelFormatEnum sdlFormat;
DrawFuncType function;
DrawFuncType functionCrop;
std::string name;
uint8_t bpp;
};
void clear();
bool prepare(AVFrame *frame, int targetWidth, int targetHeight);
bool prepareTexture(uint32_t format, int width, int height);
void rgb(AVFrame *frame);
void nv(AVFrame *frame);
void yuv(AVFrame *frame);
void crgb(AVFrame *frame);
void cnv(AVFrame *frame);
void cyuv(AVFrame *frame);
void cscale(AVFrame *frame);
void scale(AVFrame *frame);
SDL_Texture *_texture;
int _textureWidth;
int _textureHeight;
int _cropX;
int _cropY;
bool _crop;
uint8_t _bytesPerPixel;
SDL_Rect _sourceRect;
DrawFuncType _render;
SwsContext *_sws;
AVFrame *_frame;
-1
View File
@@ -33,7 +33,6 @@ public:
static inline Setting<int> fontSize{"font-size", 30};
static inline Setting<bool> vsync{"vsync", false};
static inline Setting<float> aspectCorrection{"aspect-correction", 1};
static inline Setting<int> scaler{"scaler", 2};
static inline Setting<bool> fastScale{"fast-render-scale", false};
static inline Setting<int> videoQueue{"video-buffer-size", 32};
static inline Setting<int> audioQueue{"audio-buffer-size", 16};
+1 -1
View File
@@ -68,7 +68,7 @@ public:
return item;
}
bool wait(atomic<bool> &waitFlag, int count = 0)
bool wait(atomic<bool> &waitFlag, uint8_t count = 0)
{
unique_lock<std::mutex> lock(_mtx);