Skip to content

Commit 269d70f

Browse files
XITRIXXITRIX
committed
Deko3D improvements
1 parent 56c2374 commit 269d70f

File tree

3 files changed

+84
-61
lines changed

3 files changed

+84
-61
lines changed

‎app/src/streaming/video/deko3d/DKVideoRenderer.cpp‎

Lines changed: 55 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,13 @@ namespace
5252
{
5353
static constexpr unsigned StaticCmdSize = 0x10000;
5454

55+
struct Transformation
56+
{
57+
glm::mat3 yuvmat;
58+
glm::vec3 offset;
59+
glm::vec4 uv_data;
60+
};
61+
5562
struct Vertex
5663
{
5764
float position[3];
@@ -98,7 +105,7 @@ void DKVideoRenderer::checkAndInitialize(int width, int height, AVFrame* frame)
98105
m_screen_width = width;
99106
m_screen_height = height;
100107

101-
auto *vctx = (brls::SwitchVideoContext *)brls::Application::getPlatform()->getVideoContext();
108+
vctx = (brls::SwitchVideoContext *)brls::Application::getPlatform()->getVideoContext();
102109
this->dev = vctx->getDeko3dDevice();
103110
this->queue = vctx->getQueue();
104111

@@ -123,6 +130,35 @@ void DKVideoRenderer::checkAndInitialize(int width, int height, AVFrame* frame)
123130
vertexBuffer = pool_data->allocate(sizeof(QuadVertexData), alignof(Vertex));
124131
memcpy(vertexBuffer.getCpuAddr(), QuadVertexData.data(), vertexBuffer.getSize());
125132

133+
134+
// Load the transform buffer
135+
transformUniformBuffer = pool_data->allocate(sizeof(Transformation), DK_UNIFORM_BUF_ALIGNMENT);
136+
auto transformState = reinterpret_cast<Transformation *>(transformUniformBuffer.getCpuAddr());
137+
138+
bool colorFull = frame->color_range == AVCOL_RANGE_JPEG;
139+
140+
transformState->offset = gl_color_offset(colorFull);
141+
transformState->yuvmat = gl_color_matrix(frame->colorspace, colorFull);
142+
143+
float frameAspect = ((float)m_frame_height / (float)m_frame_width);
144+
float screenAspect = ((float)m_screen_height / (float)m_screen_width);
145+
146+
if (frameAspect > screenAspect) {
147+
float multiplier = frameAspect / screenAspect;
148+
transformState->uv_data = { 0.5f - 0.5f * (1.0f / multiplier),
149+
0.0f, multiplier, 1.0f };
150+
} else {
151+
float multiplier = screenAspect / frameAspect;
152+
transformState->uv_data = { 0.0f,
153+
0.5f - 0.5f * (1.0f / multiplier), 1.0f, multiplier };
154+
}
155+
156+
157+
// cmdbuf.pushConstants(
158+
// transformUniformBuffer.getGpuAddr(), transformUniformBuffer.getSize(),
159+
// 0, sizeof(transformState), &transformState);
160+
161+
126162
// Allocate image indexes for planes
127163
lumaTextureId = vctx->allocateImageIndex();
128164
chromaTextureId = vctx->allocateImageIndex();
@@ -161,48 +197,10 @@ void DKVideoRenderer::checkAndInitialize(int width, int height, AVFrame* frame)
161197
imageDescriptorSet->update(cmdbuf, lumaTextureId, lumaDesc);
162198
imageDescriptorSet->update(cmdbuf, chromaTextureId, chromaDesc);
163199

164-
// Load the transform buffer
165-
transformUniformBuffer = pool_data->allocate(sizeof(transformState), DK_UNIFORM_BUF_ALIGNMENT);
166-
167-
bool colorFull = frame->color_range == AVCOL_RANGE_JPEG;
168-
169-
transformState.offset = gl_color_offset(colorFull);
170-
transformState.yuvmat = gl_color_matrix(frame->colorspace, colorFull);
171-
172-
float frameAspect = ((float)m_frame_height / (float)m_frame_width);
173-
float screenAspect = ((float)m_screen_height / (float)m_screen_width);
174-
175-
if (frameAspect > screenAspect) {
176-
float multiplier = frameAspect / screenAspect;
177-
transformState.uv_data = { 0.5f - 0.5f * (1.0f / multiplier),
178-
0.0f, multiplier, 1.0f };
179-
} else {
180-
float multiplier = screenAspect / frameAspect;
181-
transformState.uv_data = { 0.0f,
182-
0.5f - 0.5f * (1.0f / multiplier), 1.0f, multiplier };
183-
}
184-
185-
cmdbuf.pushConstants(
186-
transformUniformBuffer.getGpuAddr(), transformUniformBuffer.getSize(),
187-
0, sizeof(transformState), &transformState);
188-
189200
queue.submitCommands(cmdbuf.finishList());
190201
queue.waitIdle();
191202

192-
m_is_initialized = true;
193-
}
194-
195-
int frames = 0;
196-
uint64_t timeCount = 0;
197-
198-
void DKVideoRenderer::draw(NVGcontext* vg, int width, int height, AVFrame* frame, int imageFormat) {
199-
checkAndInitialize(width, height, frame);
200203

201-
uint64_t before_render = LiGetMillis();
202-
203-
if (!m_video_render_stats.rendered_frames) {
204-
m_video_render_stats.measurement_start_timestamp = before_render;
205-
}
206204

207205
dk::RasterizerState rasterizerState;
208206
dk::ColorState colorState;
@@ -212,33 +210,42 @@ void DKVideoRenderer::draw(NVGcontext* vg, int width, int height, AVFrame* frame
212210
cmdbuf.clear();
213211
cmdbuf.clearColor(0, DkColorMask_RGBA, 0.0f, 0.0f, 0.0f, 0.0f);
214212

215-
// brls::Logger::debug("TIME LOG 1: {}", float(LiGetMillis() - before_render));
216-
217213
// Bind state required for drawing the triangle
218214
cmdbuf.bindShaders(DkStageFlag_GraphicsMask, { vertexShader, fragmentShader });
219215
cmdbuf.bindTextures(DkStage_Fragment, 0, dkMakeTextureHandle(lumaTextureId, 0));
220216
cmdbuf.bindTextures(DkStage_Fragment, 1, dkMakeTextureHandle(chromaTextureId, 0));
221-
cmdbuf.bindUniformBuffer(DkStage_Vertex, 0, transformUniformBuffer.getGpuAddr(), transformUniformBuffer.getSize());
217+
cmdbuf.bindUniformBuffer(DkStage_Fragment, 2, transformUniformBuffer.getGpuAddr(), transformUniformBuffer.getSize());
222218
cmdbuf.bindRasterizerState(rasterizerState);
223219
cmdbuf.bindColorState(colorState);
224220
cmdbuf.bindColorWriteState(colorWriteState);
225221
cmdbuf.bindVtxBuffer(0, vertexBuffer.getGpuAddr(), vertexBuffer.getSize());
226222
cmdbuf.bindVtxAttribState(VertexAttribState);
227223
cmdbuf.bindVtxBufferState(VertexBufferState);
228224

229-
// brls::Logger::debug("TIME LOG 2: {}", float(LiGetMillis() - before_render));
230-
231225
// Draw the triangle
232226
cmdbuf.draw(DkPrimitive_Quads, QuadVertexData.size(), 1, 0, 0);
227+
cmdlist = cmdbuf.finishList();
233228

234-
// brls::Logger::debug("TIME LOG 3: {}", float(LiGetMillis() - before_render));
229+
m_is_initialized = true;
230+
}
231+
232+
int frames = 0;
233+
uint64_t timeCount = 0;
234+
235+
void DKVideoRenderer::draw(NVGcontext* vg, int width, int height, AVFrame* frame, int imageFormat) {
236+
checkAndInitialize(width, height, frame);
237+
238+
uint64_t before_render = LiGetMillis();
239+
240+
if (!m_video_render_stats.rendered_frames) {
241+
m_video_render_stats.measurement_start_timestamp = before_render;
242+
}
235243

236244
// Finish off this command list
237-
queue.submitCommands(cmdbuf.finishList());
245+
queue = vctx->getQueue();
246+
queue.submitCommands(cmdlist);
238247
queue.waitIdle();
239248

240-
// brls::Logger::debug("TIME LOG 4: {}", float(LiGetMillis() - before_render));
241-
242249
frames++;
243250
timeCount += LiGetMillis() - before_render;
244251

‎app/src/streaming/video/deko3d/DKVideoRenderer.hpp‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,12 @@
77
#include <glm/mat4x4.hpp>
88

99
#include <borealis.hpp>
10+
#include <borealis/platforms/switch/switch_video.hpp>
1011
#include <nanovg/framework/CShader.h>
1112
#include <nanovg/framework/CExternalImage.h>
1213
#include <nanovg/framework/CDescriptorSet.h>
1314
#include <optional>
1415

15-
struct Transformation
16-
{
17-
glm::mat3 yuvmat;
18-
glm::vec3 offset;
19-
glm::vec4 uv_data;
20-
};
21-
2216
class DKVideoRenderer : public IVideoRenderer {
2317
public:
2418
DKVideoRenderer();
@@ -38,9 +32,7 @@ class DKVideoRenderer : public IVideoRenderer {
3832
int m_screen_width = 0;
3933
int m_screen_height = 0;
4034

41-
Transformation transformState;
42-
CMemPool::Handle transformUniformBuffer;
43-
35+
brls::SwitchVideoContext *vctx = nullptr;
4436
dk::Device dev;
4537
dk::Queue queue;
4638

@@ -49,6 +41,7 @@ class DKVideoRenderer : public IVideoRenderer {
4941
std::optional<CMemPool> pool_data;
5042

5143
dk::UniqueCmdBuf cmdbuf;
44+
DkCmdList cmdlist;
5245

5346
CDescriptorSet<4096U> *imageDescriptorSet;
5447
// CDescriptorSet<1> samplerDescriptorSet;
@@ -57,6 +50,7 @@ class DKVideoRenderer : public IVideoRenderer {
5750
CShader fragmentShader;
5851

5952
CMemPool::Handle vertexBuffer;
53+
CMemPool::Handle transformUniformBuffer;
6054

6155
dk::ImageLayout lumaMappingLayout;
6256
dk::ImageLayout chromaMappingLayout;

‎app/src/streaming/video/deko3d/texture_fsh.glsl‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,29 @@ layout (std140, binding = 2) uniform Transformation
1515

1616
void main()
1717
{
18-
vec2 uv = (vTextureCoord - u.uv_data.xy) * u.uv_data.zw;
19-
vec3 YCbCr = vec3(texture(plane0, uv).r, texture(plane1, uv).r, texture(plane1, uv).g) - u.offset;
20-
outColor = vec4(clamp(u.yuvmat * YCbCr, 0.0, 1.0), 1.0);
18+
// Not work
19+
// vec2 uv = (vTextureCoord - u.uv_data.xy) * u.uv_data.zw;
20+
// vec3 YCbCr = vec3(texture(plane0, uv).r, texture(plane1, uv).r, texture(plane1, uv).g) - u.offset;
21+
// outColor = vec4(clamp(u.yuvmat * YCbCr, 0.0, 1.0), 1.0);
22+
23+
// Almost work
24+
// vec3 YCbCr = vec3(
25+
// texture2D(plane0, vTextureCoord)[0],
26+
// texture2D(plane1, vTextureCoord).xy
27+
// );
28+
29+
// YCbCr -= u.offset;
30+
// outColor = vec4(clamp(u.yuvmat * YCbCr, 0.0, 1.0), 1.0f);
31+
32+
float r, g, b, yt, ut, vt;
33+
34+
yt = texture2D(plane0, vTextureCoord).r;
35+
ut = texture2D(plane1, vTextureCoord).r - 0.5;
36+
vt = texture2D(plane1, vTextureCoord).g - 0.5;
37+
38+
r = yt + 1.13983*vt;
39+
g = yt - 0.39465*ut - 0.58060*vt;
40+
b = yt + 2.03211*ut;
41+
42+
outColor = vec4(r, g, b, 1.0);
2143
}

0 commit comments

Comments
 (0)