strepsirrhini/libreoffice-online
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make delta-builder row-based.

Browse source 
Change-Id: Ic59324535c4f412abc4e83774073eb8f57290704
This commit is contained in:
Michael Meeks 2017-09-28 09:45:46 +01:00
parent 42d264eeb0
commit 5efb59db50
2 changed files with 159 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

175
kit/Delta.hpp
View file

@ -21,106 +21,149 @@
/// A quick and dirty delta generator for last tile changes
class DeltaGenerator {
struct DeltaBitmapRow {
uint64_t _crc;
std::vector<uint32_t> _pixels;
bool identical(const DeltaBitmapRow &other) const
{
if (_crc != other._crc)
return false;
return _pixels == other._pixels;
}
};
struct DeltaData {
TileWireId _wid;
std::shared_ptr<std::vector<uint32_t>> _rawData;
int _width;
int _height;
std::vector<DeltaBitmapRow> _rows;
};
std::vector<DeltaData> _deltaEntries;
std::vector<std::shared_ptr<DeltaData>> _deltaEntries;
bool makeDelta(
const DeltaData &prevData,
const DeltaData &curData,
const DeltaData &prev,
const DeltaData &cur,
std::vector<char>& output)
{
std::vector<uint32_t> &prev = *prevData._rawData.get();
std::vector<uint32_t> &cur = *curData._rawData.get();
// FIXME: should we split and compress alpha separately ?
if (prev.size() != cur.size())
// TODO: should we split and compress alpha separately ?
if (prev._width != cur._width || prev._height != cur._height)
{
LOG_ERR("mis-sized delta: " << prev.size() << " vs " << cur.size() << "bytes");
LOG_ERR("mis-sized delta: " << prev._width << "x" << prev._height << " vs "
<< cur._width << "x" << cur._height);
return false;
}
output.push_back('D');
LOG_TRC("building delta of " << prev.size() << "bytes");
// FIXME: really lame - some RLE might help etc.
for (size_t i = 0; i < prev.size();)
{
int sameCount = 0;
while (i + sameCount < prev.size() &&
prev[i+sameCount] == cur[i+sameCount])
{
++sameCount;
}
if (sameCount > 0)
{
#if 0
if (sameCount < 64)
output.push_back(sameCount);
else
#endif
{
output.push_back(0x80 | 0x00); // long-same
output.push_back(sameCount & 0xff);
output.push_back(sameCount >> 8);
}
i += sameCount;
LOG_TRC("identical " << sameCount << "pixels");
}
LOG_TRC("building delta of a " << cur._width << "x" << cur._height << " bitmap");
int diffCount = 0;
while (i + diffCount < prev.size() &&
(prev[i+diffCount] != cur[i+diffCount]))
{
++diffCount;
}
// row move/copy src/dest is a byte.
assert (prev._height <= 256);
// column position is a byte.
assert (prev._width <= 256);
if (diffCount > 0)
// How do the rows look against each other ?
size_t lastMatchOffset = 0;
for (int y = 0; y < prev._height; ++y)
{
#if 0
if (diffCount < 64)
output.push_back(0x40 & diffCount);
else
#endif
// Life is good where rows match:
if (prev._rows[y].identical(cur._rows[y]))
continue;
// Hunt for other rows
bool matched = false;
for (int yn = 0; yn < prev._height && !matched; ++yn)
{
output.push_back(0x80 | 0x40); // long-diff
output.push_back(diffCount & 0xff);
output.push_back(diffCount >> 8);
size_t match = (y + lastMatchOffset + yn) % prev._height;
if (prev._rows[match].identical(cur._rows[y]))
{
// TODO: if offsets are >256 - use 16bits?
// hopefully find blocks of this.
lastMatchOffset = match - y;
output.push_back('c'); // copy-row
output.push_back(match); // src
output.push_back(y); // dest
matched = true;
continue;
}
}
if (matched)
continue;
// Our row is just that different:
const DeltaBitmapRow &curRow = cur._rows[y];
const DeltaBitmapRow &prevRow = prev._rows[y];
for (int x = 0; x < prev._width;)
{
int same;
for (same = 0; same + x < prev._width &&
prevRow._pixels[x+same] == curRow._pixels[x+same];)
++same;
x += same;
int diff;
for (diff = 0; diff + x < prev._width &&
(prevRow._pixels[x+diff] == curRow._pixels[x+diff] || diff < 2) &&
diff < 254;)
++diff;
if (diff > 0)
{
output.push_back('d');
output.push_back(y);
output.push_back(x);
output.push_back(diff);
size_t dest = output.size();
output.resize(dest + diffCount * 4);
memcpy(&output[dest], &cur[i], diffCount * 4);
LOG_TRC("different " << diffCount << "pixels");
i += diffCount;
output.resize(dest + diff * 4);
memcpy(&output[dest], &curRow._pixels[x], diff * 4);
LOG_TRC("different " << diff << "pixels");
x += diff;
}
}
}
return true;
}
std::shared_ptr<std::vector<uint32_t>> dataToVector(
std::shared_ptr<DeltaData> dataToDeltaData(
TileWireId wid,
unsigned char* pixmap, size_t startX, size_t startY,
int width, int height,
int bufferWidth, int bufferHeight)
{
auto data = std::make_shared<DeltaData>();
data->_wid = wid;
assert (startX + width <= (size_t)bufferWidth);
assert (startY + height <= (size_t)bufferHeight);
auto vector = std::make_shared<std::vector<uint32_t>>();
LOG_TRC("Converting data to vector of size "
LOG_TRC("Converting pixel data to delta data of size "
<< (width * height * 4) << " width " << width
<< " height " << height);
vector->resize(width * height);
data->_width = width;
data->_height = height;
data->_rows.resize(height);
for (int y = 0; y < height; ++y)
{
DeltaBitmapRow &row = data->_rows[y];
size_t position = ((startY + y) * bufferWidth * 4) + (startX * 4);
memcpy(&(*vector)[y * width], pixmap + position, width * 4);
int32_t *src = reinterpret_cast<int32_t *>(pixmap + position);
// We get the hash ~for free as we copy - with a cheap hash.
uint64_t crc = 0x7fffffff - 1;
row._pixels.resize(width);
for (int x = 0; x < width; ++x)
{
crc = (crc << 7) + crc + src[x];
row._pixels[x] = src[x];
}
}
return vector;
return data;
}
public:
@ -143,17 +186,15 @@ class DeltaGenerator {
if (_deltaEntries.size() > 6) // FIXME: hard-coded ...
_deltaEntries.erase(_deltaEntries.begin());
// FIXME: assuming width etc. are all constant & so on.
DeltaData update;
update._wid = wid;
update._rawData = dataToVector(pixmap, startX, startY, width, height,
std::shared_ptr<DeltaData> update =
dataToDeltaData(wid, pixmap, startX, startY, width, height,
bufferWidth, bufferHeight);
_deltaEntries.push_back(update);
for (auto &old : _deltaEntries)
{
if (oldWid == old._wid)
return makeDelta(old, update, output);
if (oldWid == old->_wid)
return makeDelta(*old, *update, output);
}
return false;
}

82
test/DeltaTests.cpp
View file

@ -57,61 +57,85 @@ class DeltaTests : public CPPUNIT_NS::TestFixture
const std::vector<char> &delta);
void assertEqual(const std::vector<char> &a,
const std::vector<char> &b);
const std::vector<char> &b,
int width, int height);
};
// Quick hack for debugging
std::vector<char> DeltaTests::applyDelta(
const std::vector<char> &pixmap,
png_uint_32 /* width */, png_uint_32 /* height */,
png_uint_32 width, png_uint_32 height,
const std::vector<char> &delta)
{
CPPUNIT_ASSERT(delta.size() >= 4);
CPPUNIT_ASSERT(delta[0] == 'D');
// std::cout << "apply delta of size " << delta.size() << "\n";
std::cout << "apply delta of size " << delta.size() << "\n";
// start with the same state.
std::vector<char> output = pixmap;
CPPUNIT_ASSERT_EQUAL(output.size(), pixmap.size());
size_t offset = 0;
for (size_t i = 1; i < delta.size() &&
offset < output.size();)
CPPUNIT_ASSERT_EQUAL(output.size(), size_t(pixmap.size()));
CPPUNIT_ASSERT_EQUAL(output.size(), size_t(width * height * 4));
size_t offset = 0, i;
for (i = 1; i < delta.size() && offset < output.size();)
{
bool isChangedRun = delta[i++] & 64;
CPPUNIT_ASSERT(i < delta.size());
uint32_t span = (unsigned char)delta[i++];
CPPUNIT_ASSERT(i < delta.size());
span += ((unsigned char)delta[i++])*256;
CPPUNIT_ASSERT(i < delta.size() ||
(i == delta.size() && !isChangedRun));
span *= 4;
// std::cout << "span " << span << " offset " << offset << "\n";
if (isChangedRun)
switch (delta[i])
{
CPPUNIT_ASSERT(offset + span <= output.size());
memcpy(&output[offset], &delta[i], span);
i += span;
case 'c': // copy row.
{
int srcRow = (uint8_t)(delta[i+1]);
int destRow = (uint8_t)(delta[i+2]);
std::cout << "copy row " << srcRow << " to " << destRow << "\n";
const char *src = &pixmap[width * srcRow * 4];
char *dest = &output[width * destRow * 4];
for (size_t j = 0; j < width * 4; ++j)
*dest++ = *src++;
i += 3;
break;
}
offset += span;
case 'd': // new run
{
int destRow = (uint8_t)(delta[i+1]);
int destCol = (uint8_t)(delta[i+2]);
size_t length = (uint8_t)(delta[i+3]);
i += 4;
std::cout << "new " << length << " at " << destCol << ", " << destRow << "\n";
CPPUNIT_ASSERT(length <= width - destCol);
char *dest = &output[width * destRow * 4 + destCol * 4];
for (size_t j = 0; j < length * 4 && i < delta.size(); ++j)
*dest++ = delta[i++];
break;
}
CPPUNIT_ASSERT_EQUAL(pixmap.size(), output.size());
CPPUNIT_ASSERT_EQUAL(output.size(), offset);
default:
std::cout << "Unknown delta code " << delta[i] << "\n";
CPPUNIT_ASSERT(false);
break;
}
}
CPPUNIT_ASSERT_EQUAL(delta.size(), i);
return output;
}
void DeltaTests::assertEqual(const std::vector<char> &a,
const std::vector<char> &b)
const std::vector<char> &b,
int width, int /* height */)
{
CPPUNIT_ASSERT_EQUAL(a.size(), b.size());
for (size_t i = 0; i < a.size(); ++i)
{
if (a[i] != b[i])
{
std::cout << "Differences starting at byte " << i;
std::cout << "Differences starting at byte " << i << " "
<< (i/4 % width) << ", " << (i / (width * 4)) << ":\n";
size_t len;
for (len = 0; a[i+len] != b[i+len] && i + len < a.size(); ++len)
for (len = 0; (a[i+len] != b[i+len] || len < 8) && i + len < a.size(); ++len)
{
std::cout << std::hex << (int)((unsigned char)a[i+len]) << " ";
std::cout << std::hex << (int)((unsigned char)a[i+len]) << " != ";
std::cout << std::hex << (int)((unsigned char)b[i+len]) << " ";
if (len > 0 && (len % 16 == 0))
std::cout<< "\n";
}
@ -157,7 +181,7 @@ void DeltaTests::testDeltaSequence()
// Apply it to move to the second frame
std::vector<char> reText2 = applyDelta(text, width, height, delta);
assertEqual(reText2, text2);
assertEqual(reText2, text2, width, height);
// Build a delta between text & text2Wid
std::vector<char> two2one;
@ -169,7 +193,7 @@ void DeltaTests::testDeltaSequence()
// Apply it to get back to where we started
std::vector<char> reText = applyDelta(text2, width, height, two2one);
assertEqual(reText, text);
assertEqual(reText, text, width, height);
}
void DeltaTests::testRandomDeltas()