From 4ad52186048e023234039f8344612b5123bb5700 Mon Sep 17 00:00:00 2001 From: Markus Mohrhard Date: Wed, 16 Aug 2017 03:03:04 +0200 Subject: [PATCH] introduce a way to write a simple data representation to a stream The format used is column orientated and allows quick import and export of the table content. This will be used for the external data to cache the results of each transformation step in the UI. Change-Id: I6e1bfd3b3384cbfadeb98fb995dfd0b03d5e6eb6 Reviewed-on: https://gerrit.libreoffice.org/41198 Tested-by: Jenkins Reviewed-by: Markus Mohrhard --- sc/CppunitTest_sc_cache_test.mk | 119 +++++++++++++++++++++++++++++ sc/Module_sc.mk | 1 + sc/README | 25 ++++++ sc/inc/column.hxx | 2 + sc/inc/document.hxx | 2 + sc/inc/table.hxx | 2 + sc/qa/unit/datacache.cxx | 71 +++++++++++++++++ sc/source/core/data/column4.cxx | 76 ++++++++++++++++++ sc/source/core/data/document10.cxx | 9 +++ sc/source/core/data/table7.cxx | 16 ++++ 10 files changed, 323 insertions(+) create mode 100644 sc/CppunitTest_sc_cache_test.mk create mode 100644 sc/qa/unit/datacache.cxx diff --git a/sc/CppunitTest_sc_cache_test.mk b/sc/CppunitTest_sc_cache_test.mk new file mode 100644 index 000000000000..af495db849d9 --- /dev/null +++ b/sc/CppunitTest_sc_cache_test.mk @@ -0,0 +1,119 @@ +# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*- +# +# This file is part of the LibreOffice project. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# + +$(eval $(call gb_CppunitTest_CppunitTest,sc_cache_test)) + +$(eval $(call gb_CppunitTest_use_externals,sc_cache_test, \ + boost_headers \ + icu_headers \ + icui18n \ + icuuc \ + libxml2 \ + mdds_headers \ +)) + +$(eval $(call gb_CppunitTest_add_exception_objects,sc_cache_test, \ + sc/qa/unit/datacache \ +)) + +$(eval $(call gb_CppunitTest_use_libraries,sc_cache_test, \ + basegfx \ + comphelper \ + cppu \ + cppuhelper \ + drawinglayer \ + editeng \ + for \ + forui \ + i18nlangtag \ + msfilter \ + oox \ + sal \ + salhelper \ + sax \ + sb \ + sc \ + scqahelper \ + sfx \ + sot \ + svl \ + svt \ + svx \ + svxcore \ + test \ + tk \ + tl \ + ucbhelper \ + unotest \ + utl \ + vbahelper \ + vcl \ + xo \ +)) + +$(eval $(call gb_CppunitTest_set_include,sc_cache_test,\ + -I$(SRCDIR)/sc/source/ui/inc \ + -I$(SRCDIR)/sc/inc \ + $$(INCLUDE) \ +)) + +$(eval $(call gb_CppunitTest_use_api,sc_cache_test,\ + offapi \ + oovbaapi \ + udkapi \ +)) + +$(eval $(call gb_CppunitTest_use_ure,sc_cache_test)) +$(eval $(call gb_CppunitTest_use_vcl,sc_cache_test)) + +$(eval $(call gb_CppunitTest_use_components,sc_cache_test,\ + basic/util/sb \ + chart2/source/chartcore \ + chart2/source/controller/chartcontroller \ + comphelper/util/comphelp \ + configmgr/source/configmgr \ + dbaccess/util/dba \ + embeddedobj/util/embobj \ + eventattacher/source/evtatt \ + filter/source/config/cache/filterconfig1 \ + forms/util/frm \ + framework/util/fwk \ + i18npool/util/i18npool \ + oox/util/oox \ + package/source/xstor/xstor \ + package/util/package2 \ + sax/source/expatwrap/expwrap \ + scaddins/source/analysis/analysis \ + scaddins/source/datefunc/date \ + scripting/source/basprov/basprov \ + scripting/util/scriptframe \ + sc/util/sc \ + sc/util/scd \ + sc/util/scfilt \ + $(call gb_Helper_optional,SCRIPTING, \ + sc/util/vbaobj) \ + sfx2/util/sfx \ + sot/util/sot \ + svl/source/fsstor/fsstorage \ + svl/util/svl \ + svx/util/svx \ + svx/util/svxcore \ + toolkit/util/tk \ + ucb/source/core/ucb1 \ + ucb/source/ucp/file/ucpfile1 \ + ucb/source/ucp/tdoc/ucptdoc1 \ + unotools/util/utl \ + unoxml/source/rdf/unordf \ + unoxml/source/service/unoxml \ + xmloff/util/xo \ +)) + +$(eval $(call gb_CppunitTest_use_configuration,sc_cache_test)) + +# vim: set noet sw=4 ts=4: diff --git a/sc/Module_sc.mk b/sc/Module_sc.mk index 6db1bd21224b..766069c2676d 100644 --- a/sc/Module_sc.mk +++ b/sc/Module_sc.mk @@ -45,6 +45,7 @@ $(eval $(call gb_Module_add_check_targets,sc,\ CppunitTest_sc_core \ CppunitTest_sc_dataprovider \ CppunitTest_sc_datatransformation \ + CppunitTest_sc_cache_test \ )) ifneq ($(ENABLE_HEADLESS),TRUE) diff --git a/sc/README b/sc/README index 8dfb5ec613d7..89b74f744045 100644 --- a/sc/README +++ b/sc/README @@ -14,3 +14,28 @@ Dumps the graphic objects and their position and size in pixel. Dumps the SfxItemSet representing the cell properties' of the current selection as a xml file. The file will be named dump.xml + +=== The Cache Format === + +ScDocument::StoreTabToCache allows storing the content (not the formatting) +of a table to a binary cache format. + +The format is column orientated which allows quick serialization of the table. + +Header: + * Number of Columns: 64 bit unsigned integer + +Column: + * Column Index: 64 bit unsigned integer + * Column Size: 64 bit unsigned integer + * For each cell type block a new ColumnBlock + +ColumnBlock: + * Start Row: 64 bit unsigned integer + * Block Size: 64 bit unsigned integer + * Type: 8 bit unsigned integer + - 0 : empty + - 1 : numeric + * for each cell: 64 bit IEEE 754 double precision value + - 2 : string + * for each cell: 32 bit signed string length followed by string length bytes of the string (UTF-8) diff --git a/sc/inc/column.hxx b/sc/inc/column.hxx index 35693f87aa8b..d76f3f0bd992 100644 --- a/sc/inc/column.hxx +++ b/sc/inc/column.hxx @@ -672,6 +672,8 @@ public: void EnsureFormulaCellResults( SCROW nRow1, SCROW nRow2 ); + void StoreToCache(SvStream& rStrm) const; + #if DUMP_COLUMN_STORAGE void DumpColumnStorage() const; #endif diff --git a/sc/inc/document.hxx b/sc/inc/document.hxx index f0f417baf9bd..a32eac54bf1e 100644 --- a/sc/inc/document.hxx +++ b/sc/inc/document.hxx @@ -2304,6 +2304,8 @@ public: std::unique_ptr GetColumnIterator( SCTAB nTab, SCCOL nCol, SCROW nRow1, SCROW nRow2 ) const; + SC_DLLPUBLIC void StoreTabToCache(SCTAB nTab, SvStream& rStrm) const; + #if DUMP_COLUMN_STORAGE SC_DLLPUBLIC void DumpColumnStorage( SCTAB nTab, SCCOL nCol ) const; #endif diff --git a/sc/inc/table.hxx b/sc/inc/table.hxx index 80f4b7dfc7de..a4d828c1f96a 100644 --- a/sc/inc/table.hxx +++ b/sc/inc/table.hxx @@ -1008,6 +1008,8 @@ public: void finalizeOutlineImport(); + void StoreToCache(SvStream& rStrm) const; + #if DUMP_COLUMN_STORAGE void DumpColumnStorage( SCCOL nCol ) const; #endif diff --git a/sc/qa/unit/datacache.cxx b/sc/qa/unit/datacache.cxx new file mode 100644 index 000000000000..492e31c3fa3e --- /dev/null +++ b/sc/qa/unit/datacache.cxx @@ -0,0 +1,71 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* + * This file is part of the LibreOffice project. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +#include +#include +#include +#include "helper/qahelper.hxx" + +#include "global.hxx" +#include "document.hxx" + +#include + +class ScCacheTest : public CppUnit::TestFixture +{ +public: + + void testCacheSimple(); + void testCacheString(); + + CPPUNIT_TEST_SUITE(ScCacheTest); + CPPUNIT_TEST(testCacheSimple); + CPPUNIT_TEST(testCacheString); + CPPUNIT_TEST_SUITE_END(); + +public: + virtual void setUp() override + { + utl::ConfigManager::EnableAvoidConfig(); + ScDLL::Init(); + ScGlobal::Init(); + } +}; + +void ScCacheTest::testCacheSimple() +{ + ScDocument aDoc; + aDoc.InsertTab(0, "test"); + for (SCROW nRow = 0; nRow < 10; ++nRow) + aDoc.SetValue(0, nRow, 0, nRow); + + aDoc.SetValue(0, 100000, 0, -10); + + SvMemoryStream aStrm; + aDoc.StoreTabToCache(0, aStrm); +} + +void ScCacheTest::testCacheString() +{ + ScDocument aDoc; + aDoc.InsertTab(0, "test"); + + aDoc.SetString(0, 0, 0, "TestString"); + aDoc.SetString(0, 1, 0, "asjdaonfdssda"); + aDoc.SetString(0, 2, 0, "da"); + + SvMemoryStream aStrm; + aDoc.StoreTabToCache(0, aStrm); +} + +CPPUNIT_TEST_SUITE_REGISTRATION(ScCacheTest); + +CPPUNIT_PLUGIN_IMPLEMENT(); + +/* vim:set shiftwidth=4 softtabstop=4 expandtab: */ diff --git a/sc/source/core/data/column4.cxx b/sc/source/core/data/column4.cxx index 7132ed02f732..e7104a856d99 100644 --- a/sc/source/core/data/column4.cxx +++ b/sc/source/core/data/column4.cxx @@ -1632,4 +1632,80 @@ void ScColumn::EnsureFormulaCellResults( SCROW nRow1, SCROW nRow2 ) ); } +namespace { + +class StoreToCacheFunc +{ + SvStream& mrStrm; +public: + + StoreToCacheFunc(SvStream& rStrm): + mrStrm(rStrm) + { + } + + void operator() ( const sc::CellStoreType::value_type& node, size_t nOffset, size_t nDataSize ) + { + SCROW nStartRow = node.position + nOffset; + mrStrm.WriteUInt64(nStartRow); + mrStrm.WriteUInt64(nDataSize); + switch (node.type) + { + case sc::element_type_empty: + { + mrStrm.WriteUChar(0); + } + break; + case sc::element_type_numeric: + { + mrStrm.WriteUChar(1); + sc::numeric_block::const_iterator it = sc::numeric_block::begin(*node.data); + std::advance(it, nOffset); + sc::numeric_block::const_iterator itEnd = it; + std::advance(itEnd, nDataSize); + + for (; it != itEnd; ++it) + { + mrStrm.WriteDouble(*it); + } + } + break; + case sc::element_type_string: + { + mrStrm.WriteUChar(2); + sc::string_block::const_iterator it = sc::string_block::begin(*node.data); + std::advance(it, nOffset); + sc::string_block::const_iterator itEnd = it; + std::advance(itEnd, nDataSize); + + for (; it != itEnd; ++it) + { + OString aStr = OUStringToOString(it->getString(), RTL_TEXTENCODING_UTF8); + sal_Int32 nStrLength = aStr.getLength(); + mrStrm.WriteInt32(nStrLength); + mrStrm.WriteCharPtr(aStr.getStr()); + } + } + break; + case sc::element_type_formula: + { + mrStrm.WriteUChar(2); + } + break; + } + } +}; + +} + +void ScColumn::StoreToCache(SvStream& rStrm) const +{ + rStrm.WriteUInt64(nCol); + SCROW nLastRow = GetLastDataPos(); + rStrm.WriteUInt64(nLastRow + 1); // the rows are zero based + + StoreToCacheFunc aFunc(rStrm); + sc::ParseBlock(maCells.begin(), maCells, aFunc, (SCROW)0, nLastRow); +} + /* vim:set shiftwidth=4 softtabstop=4 expandtab: */ diff --git a/sc/source/core/data/document10.cxx b/sc/source/core/data/document10.cxx index 89c1975ab0cb..bd6f6e862cf5 100644 --- a/sc/source/core/data/document10.cxx +++ b/sc/source/core/data/document10.cxx @@ -939,4 +939,13 @@ sc::ExternalDataMapper& ScDocument::GetExternalDataMapper() return *mpDataMapper; } +void ScDocument::StoreTabToCache(SCTAB nTab, SvStream& rStrm) const +{ + const ScTable* pTab = FetchTable(nTab); + if (!pTab) + return; + + pTab->StoreToCache(rStrm); +} + /* vim:set shiftwidth=4 softtabstop=4 expandtab: */ diff --git a/sc/source/core/data/table7.cxx b/sc/source/core/data/table7.cxx index e20f5b44d777..340dc7476fae 100644 --- a/sc/source/core/data/table7.cxx +++ b/sc/source/core/data/table7.cxx @@ -417,4 +417,20 @@ void ScTable::finalizeOutlineImport() } } +void ScTable::StoreToCache(SvStream& rStrm) const +{ + SCCOL nStartCol = 0; + SCCOL nEndCol = MAXCOL; + SCROW nStartRow = 0; + SCROW nEndRow = MAXROW; + + GetDataArea(nStartCol, nStartRow, nEndCol, nEndRow, false, false); + + rStrm.WriteUInt64(nEndCol + 1); + for (SCCOL nCol = 0; nCol <= nEndCol; ++nCol) + { + aCol[nCol].StoreToCache(rStrm); + } +} + /* vim:set shiftwidth=4 softtabstop=4 expandtab: */