Skip to content
Snippets Groups Projects
Commit 6f2256f2 authored by Hong-Chang Shin's avatar Hong-Chang Shin Committed by Bart Kroon
Browse files

m58035 integration

parent e4d46f87
No related branches found
No related tags found
No related merge requests found
......@@ -40,6 +40,7 @@
"depthParameter": 50,
"dilate": 5,
"enable2ndPassPruner": true,
"sampleSize": 32,
"erode": 2,
"maxBasicViewsPerGraph": 3,
"maxColorError": 0.1,
......
......@@ -24,6 +24,7 @@
"depthParameter": 50,
"dilate": 5,
"enable2ndPassPruner": true,
"sampleSize": 32,
"erode": 2,
"maxBasicViewsPerGraph": 3,
"maxColorError": 0.1,
......
......@@ -39,6 +39,7 @@
"depthParameter": 50,
"dilate": 5,
"enable2ndPassPruner": true,
"sampleSize": 32,
"erode": 2,
"maxBasicViewsPerGraph": 3,
"maxColorError": 0.1,
......
......@@ -40,6 +40,7 @@
"depthParameter": 50,
"dilate": 5,
"enable2ndPassPruner": true,
"sampleSize": 32,
"erode": 2,
"maxBasicViewsPerGraph": 3,
"maxColorError": 0.1,
......
......@@ -22,6 +22,7 @@
"depthParameter": 50,
"dilate": 5,
"enable2ndPassPruner": true,
"sampleSize": 32,
"erode": 2,
"maxBasicViewsPerGraph": 3,
"maxColorError": 0.1,
......
......@@ -40,6 +40,7 @@
"depthParameter": 50,
"dilate": 5,
"enable2ndPassPruner": true,
"sampleSize": 32,
"erode": 2,
"maxBasicViewsPerGraph": 3,
"maxColorError": 0.1,
......
......@@ -136,6 +136,8 @@ private:
const int32_t m_dilate{};
const int32_t m_maxBasicViewsPerGraph{};
const bool m_enable2ndPassPruner{};
const int32_t m_sampleSize{};
int32_t m_reviveRatio{100};
const float m_maxColorError{};
const Renderer::AccumulatingPixel<Common::Vec3f> m_config;
std::optional<float> m_lumaStdDev{};
......@@ -160,6 +162,7 @@ public:
, m_dilate{nodeConfig.require("dilate").as<int32_t>()}
, m_maxBasicViewsPerGraph{nodeConfig.require("maxBasicViewsPerGraph").as<int32_t>()}
, m_enable2ndPassPruner{nodeConfig.require("enable2ndPassPruner").as<bool>()}
, m_sampleSize{nodeConfig.require("sampleSize").as<int32_t>()}
, m_maxColorError{nodeConfig.require("maxColorError").as<float>()}
, m_config{nodeConfig.require("rayAngleParameter").as<float>(),
nodeConfig.require("depthParameter").as<float>(),
......@@ -421,6 +424,7 @@ public:
private:
void analyzeFillAndPruneAgain(const Common::MVD16Frame &views, int32_t nonPrunedArea,
int32_t percentageRatio) {
const float A = 0.5F / (1.F - m_lumaStdDev.value());
std::cout << "Pruning luma threshold: " << (m_lumaStdDev.value() * m_maxLumaError) << "\n";
while (nonPrunedArea > (m_params.sampleBudget * percentageRatio / 100) &&
......@@ -433,10 +437,13 @@ private:
if (m_lumaStdDev.value() > 1.0F) {
m_lumaStdDev.emplace(1.0F);
}
m_reviveRatio = static_cast<int>(100.F * ((1.F - m_lumaStdDev.value()) * A + 0.5F));
prepareFrame(views);
nonPrunedArea = pruneFrame(views);
std::cout << "Pruning luma threshold: " << (m_lumaStdDev.value() * m_maxLumaError) << "\n";
std::cout << "reviveRatio: " << m_reviveRatio << "\n";
}
}
......@@ -609,6 +616,144 @@ private:
return result;
}
static void calcSamples(const int32_t sampleSize, const std::vector<size_t> &nonPrunedPixIndices,
const TMIV::Common::Mat<TMIV::Common::Vec3f> &referenceRGB,
const TMIV::Common::Mat<TMIV::Common::Vec3f> &synthesizedRGB,
std::vector<uint64_t> &sampledIndices, std::vector<float> &weights) {
struct VecRgb {
VecRgb(float _r, float _g, float _b) : r{_r}, g{_g}, b{_b} {}
float r{}, g{}, b{};
};
std::vector<std::vector<uint64_t>> sampledPixIndices(sampleSize * sampleSize * sampleSize);
std::vector<VecRgb> meanColors(sampleSize * sampleSize * sampleSize, {0.F, 0.F, 0.F});
for (size_t idx : nonPrunedPixIndices) {
VecRgb ref{std::min(255.F, std::max(0.F, referenceRGB[idx][0]) * 255.F),
std::min(255.F, std::max(0.F, referenceRGB[idx][1]) * 255.F),
std::min(255.F, std::max(0.F, referenceRGB[idx][2]) * 255.F)};
VecRgb synth{synthesizedRGB[idx][0] * 255.F, synthesizedRGB[idx][1] * 255.F,
synthesizedRGB[idx][2] * 255.F};
int32_t idxBin =
std::min(sampleSize - 1, static_cast<int32_t>(ref.r) * sampleSize / 256) * sampleSize *
sampleSize +
std::min(sampleSize - 1, static_cast<int32_t>(ref.g) * sampleSize / 256) * sampleSize +
std::min(sampleSize - 1, static_cast<int32_t>(ref.b) * sampleSize / 256);
sampledPixIndices[idxBin].push_back(idx);
meanColors[idxBin].r += static_cast<float>(static_cast<int32_t>(synth.r));
meanColors[idxBin].g += static_cast<float>(static_cast<int32_t>(synth.g));
meanColors[idxBin].b += static_cast<float>(static_cast<int32_t>(synth.b));
}
for (int32_t i = 0; i < sampleSize * sampleSize * sampleSize; i++) {
float minv = 3.F * 255.F * 255.F;
size_t minIdx = sampledPixIndices[i].size();
meanColors[i].r /= static_cast<float>(sampledPixIndices[i].size());
meanColors[i].g /= static_cast<float>(sampledPixIndices[i].size());
meanColors[i].b /= static_cast<float>(sampledPixIndices[i].size());
for (size_t j = 0; j < sampledPixIndices[i].size(); j++) {
size_t idx = sampledPixIndices[i][j];
VecRgb d{meanColors[i].r - synthesizedRGB[idx][0] * 255.F,
meanColors[i].g - synthesizedRGB[idx][1] * 255.F,
meanColors[i].b - synthesizedRGB[idx][2] * 255.F};
auto err = d.r * d.r + d.g * d.g + d.b * d.b;
if (minv > err) {
minv = err;
minIdx = j;
}
}
if (minIdx < sampledPixIndices[i].size()) {
sampledIndices.push_back(sampledPixIndices[i][minIdx]);
weights.push_back(static_cast<float>(sampledPixIndices[i].size()) /
static_cast<float>(nonPrunedPixIndices.size()));
}
}
}
[[nodiscard]] auto
getColorInconsistencyMaskSampled(const Common::Mat<Common::Vec3f> &referenceYUV,
const Common::Mat<Common::Vec3f> &synthesizedYUV,
const Common::Mat<uint8_t> &prunedMask) const
-> Common::Mat<uint8_t> {
struct VecRgb {
VecRgb(float _r, float _g, float _b) : r{_r}, g{_g}, b{_b} {}
float r{}, g{}, b{};
};
const double eps = 1E-10;
Common::Mat<uint8_t> result(prunedMask.sizes(), 0);
const auto referenceRGB{createRgbImageFromYuvImage(referenceYUV)};
const auto synthesizedRGB{createRgbImageFromYuvImage(synthesizedYUV)};
const auto nonPrunedPixIndices{computeNonPrunedPixelIndices(synthesizedYUV, prunedMask)};
if (nonPrunedPixIndices.empty()) {
return result;
}
std::vector<uint64_t> sampledIndices;
std::vector<float> weights;
calcSamples(m_sampleSize, nonPrunedPixIndices, referenceRGB, synthesizedRGB, sampledIndices,
weights);
const auto x1{iterativeReweightedLeastSquaresOnNonPrunedPixels(
sampledIndices, referenceRGB, synthesizedRGB, weights, 0, eps)};
const auto x2{iterativeReweightedLeastSquaresOnNonPrunedPixels(
sampledIndices, referenceRGB, synthesizedRGB, weights, 1, eps)};
const auto x3{iterativeReweightedLeastSquaresOnNonPrunedPixels(
sampledIndices, referenceRGB, synthesizedRGB, weights, 2, eps)};
std::vector<int> nonPrunedPixErrBinIdx(nonPrunedPixIndices.size());
const int32_t numBinErrHistogram = 100;
double errHistThreshExeedSampleNum = 0.F;
std::vector<double> errHistogram(numBinErrHistogram, 0.F);
for (size_t i = 0; i < nonPrunedPixIndices.size(); ++i) {
size_t pixIdx = nonPrunedPixIndices[i];
const VecRgb s{synthesizedRGB[pixIdx][0], synthesizedRGB[pixIdx][1],
synthesizedRGB[pixIdx][2]};
const VecRgb c{static_cast<float>(computeWeightedRgbSum(referenceRGB[pixIdx], x1)),
static_cast<float>(computeWeightedRgbSum(referenceRGB[pixIdx], x2)),
static_cast<float>(computeWeightedRgbSum(referenceRGB[pixIdx], x3))};
VecRgb d{std::abs(c.r - s.r), std::abs(c.g - s.g), std::abs(c.b - s.b)};
auto err = std::max(d.r, std::max(d.g, d.b));
int32_t errHistBinIdx = std::min(
numBinErrHistogram - 1, static_cast<int>(err * static_cast<float>(numBinErrHistogram)));
nonPrunedPixErrBinIdx[i] = errHistBinIdx;
errHistogram[errHistBinIdx] += 1.F;
if (err > m_maxColorError) {
errHistThreshExeedSampleNum += 1.F;
if (m_reviveRatio == 100) {
result[pixIdx] = 255;
}
}
}
if (m_reviveRatio == 100) {
return result;
}
int32_t cutBinIdx = numBinErrHistogram;
float accumHist = 0.F;
float minimumErr = 1.F / static_cast<float>(numBinErrHistogram);
for (int32_t i = numBinErrHistogram - 1; i >= 0; i--) {
accumHist += static_cast<float>(errHistogram[i] / errHistThreshExeedSampleNum);
double curErr = minimumErr * static_cast<float>(i);
if (accumHist < static_cast<float>(m_reviveRatio) / 100.F && curErr >= m_maxColorError) {
cutBinIdx = i;
} else {
break;
}
}
for (size_t i = 0; i < nonPrunedPixIndices.size(); i++) {
size_t pixIdx = nonPrunedPixIndices[i];
if (nonPrunedPixErrBinIdx[i] >= cutBinIdx) {
result[pixIdx] = 255;
}
}
return result;
}
[[nodiscard]] auto getColorInconsistencyMask(const Common::Mat<Common::Vec3f> &referenceYUV,
const Common::Mat<Common::Vec3f> &synthesizedYUV,
const Common::Mat<uint8_t> &prunedMask) const
......@@ -699,8 +844,13 @@ private:
Common::Mat<uint8_t> colorInconsistencyMask;
Common::Mat<uint8_t>::iterator iColor;
if (m_enable2ndPassPruner) {
colorInconsistencyMask = getColorInconsistencyMask(
synthesizer.referenceYUV, synthesizer.rasterizer.attribute<0>(), mask);
if (m_sampleSize == 0) {
colorInconsistencyMask = getColorInconsistencyMask(
synthesizer.referenceYUV, synthesizer.rasterizer.attribute<0>(), mask);
} else {
colorInconsistencyMask = getColorInconsistencyMaskSampled(
synthesizer.referenceYUV, synthesizer.rasterizer.attribute<0>(), mask);
}
iColor = std::begin(colorInconsistencyMask);
}
......
5b160bc68cbb408089a6c79976395d72 *A3/E/QP3/A3_E_QP3_v11_tex_480x270_yuv420p10le.yuv
c17151b319bfc2185efca1ce14ccbd38 *A3/E/QP3/TMIV_A3_E_QP3.bit
c2817a9ac8a3f2308b179fc253ba893f *A3/E/QP3/TMIV_A3_E_QP3.dec
f66195942470e64f7ea156ee46de2f30 *A3/E/QP3/TMIV_A3_E_QP3.hls
63b6463d22c5b80520ab86dbbf1dce86 *A3/E/QP3/TMIV_A3_E_QP3_geo_c00.bit
26349e6e50fc0733837eca929fcf35a5 *A3/E/QP3/TMIV_A3_E_QP3_geo_c01.bit
10d5465b69e0742742f217911a0072b7 *A3/E/QP3/TMIV_A3_E_QP3_tex_c00.bit
29ea425ac0e6e678036bcbccf8fbb089 *A3/E/QP3/TMIV_A3_E_QP3_tex_c01.bit
6fe3e5defcfbfffe3f6a5bf7a4e464bd *A3/E/TMIV_A3_E.bit
4eb35e59c2debd6ae70cf33ec988be1f *A3/E/TMIV_A3_E_geo_c00_512x512_yuv420p10le.yuv
63fcebf9a288e9331b2dab7d70f1b6e8 *A3/E/TMIV_A3_E_geo_c01_512x512_yuv420p10le.yuv
3f52ed3bec5a66c6da112bc21417fd70 *A3/E/TMIV_A3_E_tex_c00_1024x1024_yuv420p10le.yuv
c6c8b247b2ece86d3c39f4bb906d7015 *A3/E/TMIV_A3_E_tex_c01_1024x1024_yuv420p10le.yuv
4e2b717b0597d56f303aceaca50fb0fd *E3/B/QP3/E3_B_QP3_p01_tex_512x512_yuv420p10le.yuv
dbb46e4db00e2c05e770d270b34db330 *E3/B/QP3/TMIV_E3_B_QP3.bit
952c8fcffa4947a28e768cd2f21477b4 *E3/B/QP3/TMIV_E3_B_QP3_geo_c00.bit
ba698cdc3fdbf422e586d1e2a0deb240 *E3/B/QP3/TMIV_E3_B_QP3_tex_c00.bit
ed341f46382bb16808317edf63aacb7c *E3/B/TMIV_E3_B.bit
c8045e586557d0351e55e44866707298 *E3/B/TMIV_E3_B_geo_c00_256x1024_yuv420p10le.yuv
c2094d5d38c3cddb91219afb4d4698a6 *E3/B/TMIV_E3_B_tex_c00_512x2048_yuv420p10le.yuv
bd6d9e6b1bff5adb20d863093ec4713e *A3/E/QP3/A3_E_QP3_v11_tex_480x270_yuv420p10le.yuv
cecf70edd0be68c86b59582cec9d1bb3 *A3/E/QP3/TMIV_A3_E_QP3.bit
1c08e7169f646dd545457d28e49344d9 *A3/E/QP3/TMIV_A3_E_QP3.dec
be9a7381a755be7ba2068d5630f65f32 *A3/E/QP3/TMIV_A3_E_QP3.hls
a01ed492bba2536470658f9861641f7d *A3/E/QP3/TMIV_A3_E_QP3_geo_c00.bit
fb3d1b8a06014f6dde06ad86a8af38aa *A3/E/QP3/TMIV_A3_E_QP3_geo_c01.bit
79059c04dc4d3465df20963225169c8b *A3/E/QP3/TMIV_A3_E_QP3_tex_c00.bit
07b6aba92510dab4880219b14a6abf1c *A3/E/QP3/TMIV_A3_E_QP3_tex_c01.bit
6a03c71b17624401324d1d5225a14a20 *A3/E/TMIV_A3_E.bit
c4703e0798759acfa1c98a2a9625af24 *A3/E/TMIV_A3_E_geo_c00_512x512_yuv420p10le.yuv
2ed8f8ffac90443a4188ced89a914080 *A3/E/TMIV_A3_E_geo_c01_512x512_yuv420p10le.yuv
5146ef3381a3fcda77b2f3f01cdc1205 *A3/E/TMIV_A3_E_tex_c00_1024x1024_yuv420p10le.yuv
4b8c96b1a2cec11f23ca8c4d4ff30760 *A3/E/TMIV_A3_E_tex_c01_1024x1024_yuv420p10le.yuv
52911f2ad3930ba4bc434d048ba0b710 *E3/B/QP3/E3_B_QP3_p01_tex_512x512_yuv420p10le.yuv
f0aaabda530d8a15e3e668600424ee38 *E3/B/QP3/TMIV_E3_B_QP3.bit
2b674a99c65343d2d7192a28e256d910 *E3/B/QP3/TMIV_E3_B_QP3_geo_c00.bit
1c52e186d9f35cd55b7121e6b38167ed *E3/B/QP3/TMIV_E3_B_QP3_tex_c00.bit
7a7bd5816968d10a34cc79f4119dbed4 *E3/B/TMIV_E3_B.bit
59ffc01ffb8e06274c2e1a35f5c07efe *E3/B/TMIV_E3_B_geo_c00_256x1024_yuv420p10le.yuv
c3774dfd281b62a1e93c1da88bd93097 *E3/B/TMIV_E3_B_tex_c00_512x2048_yuv420p10le.yuv
61882a3bc1dc8cd9437d02a903a588da *G3/N/R0/TMIV_G3_N_R0_0000.json
d2eeae0932b69461578b57a0bb6d0b39 *G3/N/R0/TMIV_G3_N_R0_tex_pv00_512x512_yuv420p10le.yuv
cf29cb6200794bdbf82abb512bc0b152 *G3/N/R0/TMIV_G3_N_R0_tex_pv01_512x512_yuv420p10le.yuv
......@@ -40,20 +40,20 @@ b99b264f0f0681f991c1e53eb91d305d *M3/M/QP3/TMIV_M3_M_QP3_tra_c00.bit
b948ece26f7d6fc31262919f38e393e4 *M3/M/TMIV_M3_M.bit
689d17bd09873c14a9cab16f4e2714dc *M3/M/TMIV_M3_M_tex_c00_1280x1280_yuv420p10le.yuv
5725cacd49b151f36f0aabe524c9005a *M3/M/TMIV_M3_M_tra_c00_1280x1280_yuv420p10le.yuv
d3d39b51d27a262bb2eef8913283188d *O3/N/R0/O3_N_R0_p01_tex_512x512_yuv420p10le.yuv
0886cc8d870b73d5f72717dc3e4cdd42 *O3/N/TMIV_O3_N.bit
6b8670edab3d18daffab0b595eb62153 *O3/N/TMIV_O3_N.hls
8326312644bea20a3fddd563c0655a74 *O3/N/TMIV_O3_N_geo_c00_512x640_yuv420p10le.yuv
badec6c9560f652f652feb59bf74a10a *O3/N/TMIV_O3_N_geo_c01_512x640_yuv420p10le.yuv
dd6076ccbf6ec4606c2257c3b0218449 *O3/N/R0/O3_N_R0_p01_tex_512x512_yuv420p10le.yuv
092b03619ba92cf23566405f4ed0f02b *O3/N/TMIV_O3_N.bit
30976509889273529065a2334ff45543 *O3/N/TMIV_O3_N.hls
a1f5bc8ae88436a41c4e52044e1a9879 *O3/N/TMIV_O3_N_geo_c00_512x640_yuv420p10le.yuv
f5cb9b80a2da5d853da60a89a006818d *O3/N/TMIV_O3_N_geo_c01_512x640_yuv420p10le.yuv
1de085bace14a557c6db835f68199d7b *O3/N/TMIV_O3_N_occ_c00_32x40_yuv420p10le.yuv
2cf02402e21aa7d64af179ecdcc75547 *O3/N/TMIV_O3_N_occ_c01_32x40_yuv420p10le.yuv
4c2ef5540170644b03bf5b4b2bd4e393 *O3/N/TMIV_O3_N_tex_c00_1024x1280_yuv420p10le.yuv
450efe3ba5205d0af6e780e691b754c9 *O3/N/TMIV_O3_N_tex_c01_1024x1280_yuv420p10le.yuv
93353cebe9cb6e86c2a1d61a9f6c6dba *P3/E/R0/P3_E_R0_v11_tex_480x270_yuv420p10le.yuv
e5e9b6a0836c1bbe8541274fa3b5fbd6 *P3/E/TMIV_P3_E.bit
2e3dda3074364dd1247477a682312c61 *P3/E/TMIV_P3_E.hls
bb300dec5921dc3e16784aab9fc7851a *P3/E/TMIV_P3_E_pac_c00_1024x1280_yuv420p10le.yuv
e19f87f8a73f9d8edef6806274845ab9 *P3/E/TMIV_P3_E_pac_c01_1024x1280_yuv420p10le.yuv
562bca6bedc02f5cfb324373c674a3d4 *O3/N/TMIV_O3_N_occ_c01_32x40_yuv420p10le.yuv
c63601065829ebaa94a8a176fa3b62a1 *O3/N/TMIV_O3_N_tex_c00_1024x1280_yuv420p10le.yuv
e9f4010e425acb6605d044e38b03bf98 *O3/N/TMIV_O3_N_tex_c01_1024x1280_yuv420p10le.yuv
e17ff49acf00eb0ae32601378c866c74 *P3/E/R0/P3_E_R0_v11_tex_480x270_yuv420p10le.yuv
2b80be97262fc1e05b0ac49531bcd821 *P3/E/TMIV_P3_E.bit
01c087ff6a48ea816fd5423d2bb8fb0e *P3/E/TMIV_P3_E.hls
8f98d6d50f2c096611ba7dba5589019d *P3/E/TMIV_P3_E_pac_c00_1024x1280_yuv420p10le.yuv
ac29521a42a7a4e54106b4be9e1b8721 *P3/E/TMIV_P3_E_pac_c01_1024x1280_yuv420p10le.yuv
3ec2bbc7d729ea8181aa170d31a43c7f *R3/O/R0/R3_O_R0_p02_geo_480x270_yuv420p16le.yuv
0b01ceb6c22a1de270ea35a7e30c0f42 *R3/O/R0/R3_O_R0_p02_tex_480x270_yuv420p10le.yuv
e78d249a4aba45394d3c26a3fb15374a *R3/O/R0/R3_O_R0_v01_geo_480x270_yuv420p16le.yuv
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment