introduce a way to write a simple data representation to a stream

The format used is column orientated and allows quick import and
export of the table content. This will be used for the external data
to cache the results of each transformation step in the UI.

Change-Id: I6e1bfd3b3384cbfadeb98fb995dfd0b03d5e6eb6
Reviewed-on: https://gerrit.libreoffice.org/41198
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Markus Mohrhard <markus.mohrhard@googlemail.com>

sc/CppunitTest_sc_cache_test.mk

@@ -0,0 +1,119 @@
+# -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+$(eval $(call gb_CppunitTest_CppunitTest,sc_cache_test))
+
+$(eval $(call gb_CppunitTest_use_externals,sc_cache_test, \
+	boost_headers \
+    icu_headers \
+    icui18n \
+    icuuc \
+	libxml2 \
+	mdds_headers \
+))
+
+$(eval $(call gb_CppunitTest_add_exception_objects,sc_cache_test, \
+    sc/qa/unit/datacache \
+))
+
+$(eval $(call gb_CppunitTest_use_libraries,sc_cache_test, \
+    basegfx \
+    comphelper \
+    cppu \
+    cppuhelper \
+    drawinglayer \
+    editeng \
+    for \
+    forui \
+    i18nlangtag \
+    msfilter \
+    oox \
+    sal \
+    salhelper \
+    sax \
+    sb \
+    sc \
+    scqahelper \
+    sfx \
+    sot \
+    svl \
+    svt \
+    svx \
+    svxcore \
+	test \
+    tk \
+    tl \
+    ucbhelper \
+	unotest \
+    utl \
+    vbahelper \
+    vcl \
+    xo \
+))
+
+$(eval $(call gb_CppunitTest_set_include,sc_cache_test,\
+    -I$(SRCDIR)/sc/source/ui/inc \
+    -I$(SRCDIR)/sc/inc \
+    $$(INCLUDE) \
+))
+
+$(eval $(call gb_CppunitTest_use_api,sc_cache_test,\
+    offapi \
+    oovbaapi \
+    udkapi \
+))
+
+$(eval $(call gb_CppunitTest_use_ure,sc_cache_test))
+$(eval $(call gb_CppunitTest_use_vcl,sc_cache_test))
+
+$(eval $(call gb_CppunitTest_use_components,sc_cache_test,\
+    basic/util/sb \
+    chart2/source/chartcore \
+    chart2/source/controller/chartcontroller \
+    comphelper/util/comphelp \
+    configmgr/source/configmgr \
+    dbaccess/util/dba \
+    embeddedobj/util/embobj \
+    eventattacher/source/evtatt \
+    filter/source/config/cache/filterconfig1 \
+    forms/util/frm \
+    framework/util/fwk \
+    i18npool/util/i18npool \
+    oox/util/oox \
+    package/source/xstor/xstor \
+    package/util/package2 \
+    sax/source/expatwrap/expwrap \
+    scaddins/source/analysis/analysis \
+    scaddins/source/datefunc/date \
+    scripting/source/basprov/basprov \
+    scripting/util/scriptframe \
+    sc/util/sc \
+    sc/util/scd \
+    sc/util/scfilt \
+    $(call gb_Helper_optional,SCRIPTING, \
+	    sc/util/vbaobj) \
+    sfx2/util/sfx \
+    sot/util/sot \
+    svl/source/fsstor/fsstorage \
+    svl/util/svl \
+    svx/util/svx \
+    svx/util/svxcore \
+    toolkit/util/tk \
+    ucb/source/core/ucb1 \
+    ucb/source/ucp/file/ucpfile1 \
+    ucb/source/ucp/tdoc/ucptdoc1 \
+    unotools/util/utl \
+    unoxml/source/rdf/unordf \
+    unoxml/source/service/unoxml \
+    xmloff/util/xo \
+))
+
+$(eval $(call gb_CppunitTest_use_configuration,sc_cache_test))
+
+# vim: set noet sw=4 ts=4:

sc/Module_sc.mk

@@ -45,6 +45,7 @@ $(eval $(call gb_Module_add_check_targets,sc,\
 	CppunitTest_sc_core \
 	CppunitTest_sc_dataprovider \
 	CppunitTest_sc_datatransformation \
+	CppunitTest_sc_cache_test \
 ))
 
 ifneq ($(ENABLE_HEADLESS),TRUE)

sc/README

@@ -14,3 +14,28 @@ Dumps the graphic objects and their position and size in pixel.
 
 Dumps the SfxItemSet representing the cell properties' of the
 current selection as a xml file. The file will be named dump.xml
+
+=== The Cache Format ===
+
+ScDocument::StoreTabToCache allows storing the content (not the formatting)
+of a table to a binary cache format.
+
+The format is column orientated which allows quick serialization of the table.
+
+Header:
+    * Number of Columns: 64 bit unsigned integer
+
+Column:
+    * Column Index: 64 bit unsigned integer
+    * Column Size: 64 bit unsigned integer
+    * For each cell type block a new ColumnBlock
+
+ColumnBlock:
+    * Start Row: 64 bit unsigned integer
+    * Block Size: 64 bit unsigned integer
+    * Type: 8 bit unsigned integer
+        - 0 : empty
+        - 1 : numeric
+            * for each cell: 64 bit IEEE 754 double precision value
+        - 2 : string
+            * for each cell: 32 bit signed string length followed by string length bytes of the string (UTF-8)

sc/inc/column.hxx

@@ -672,6 +672,8 @@ public:
 
     void EnsureFormulaCellResults( SCROW nRow1, SCROW nRow2 );
 
+    void StoreToCache(SvStream& rStrm) const;
+
 #if DUMP_COLUMN_STORAGE
     void DumpColumnStorage() const;
 #endif

sc/inc/document.hxx

@@ -2304,6 +2304,8 @@ public:
 
     std::unique_ptr<sc::ColumnIterator> GetColumnIterator( SCTAB nTab, SCCOL nCol, SCROW nRow1, SCROW nRow2 ) const;
 
+    SC_DLLPUBLIC void StoreTabToCache(SCTAB nTab, SvStream& rStrm) const;
+
 #if DUMP_COLUMN_STORAGE
     SC_DLLPUBLIC void DumpColumnStorage( SCTAB nTab, SCCOL nCol ) const;
 #endif

sc/inc/table.hxx

@@ -1008,6 +1008,8 @@ public:
 
     void finalizeOutlineImport();
 
+    void StoreToCache(SvStream& rStrm) const;
+
 #if DUMP_COLUMN_STORAGE
     void DumpColumnStorage( SCCOL nCol ) const;
 #endif

sc/qa/unit/datacache.cxx

@@ -0,0 +1,71 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+#include <sal/config.h>
+#include <test/bootstrapfixture.hxx>
+#include <unotools/configmgr.hxx>
+#include "helper/qahelper.hxx"
+
+#include "global.hxx"
+#include "document.hxx"
+
+#include <tools/stream.hxx>
+
+class ScCacheTest : public CppUnit::TestFixture
+{
+public:
+
+    void testCacheSimple();
+    void testCacheString();
+
+    CPPUNIT_TEST_SUITE(ScCacheTest);
+    CPPUNIT_TEST(testCacheSimple);
+    CPPUNIT_TEST(testCacheString);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    virtual void setUp() override
+    {
+        utl::ConfigManager::EnableAvoidConfig();
+        ScDLL::Init();
+        ScGlobal::Init();
+    }
+};
+
+void ScCacheTest::testCacheSimple()
+{
+    ScDocument aDoc;
+    aDoc.InsertTab(0, "test");
+    for (SCROW nRow = 0; nRow < 10; ++nRow)
+        aDoc.SetValue(0, nRow, 0, nRow);
+
+    aDoc.SetValue(0, 100000, 0, -10);
+
+    SvMemoryStream aStrm;
+    aDoc.StoreTabToCache(0, aStrm);
+}
+
+void ScCacheTest::testCacheString()
+{
+    ScDocument aDoc;
+    aDoc.InsertTab(0, "test");
+
+    aDoc.SetString(0, 0, 0, "TestString");
+    aDoc.SetString(0, 1, 0, "asjdaonfdssda");
+    aDoc.SetString(0, 2, 0, "da");
+
+    SvMemoryStream aStrm;
+    aDoc.StoreTabToCache(0, aStrm);
+}
+
+CPPUNIT_TEST_SUITE_REGISTRATION(ScCacheTest);
+
+CPPUNIT_PLUGIN_IMPLEMENT();
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */

sc/source/core/data/column4.cxx

@@ -1632,4 +1632,80 @@ void ScColumn::EnsureFormulaCellResults( SCROW nRow1, SCROW nRow2 )
     );
 }
 
+namespace {
+
+class StoreToCacheFunc
+{
+    SvStream& mrStrm;
+public:
+
+    StoreToCacheFunc(SvStream& rStrm):
+        mrStrm(rStrm)
+    {
+    }
+
+    void operator() ( const sc::CellStoreType::value_type& node, size_t nOffset, size_t nDataSize )
+    {
+        SCROW nStartRow = node.position + nOffset;
+        mrStrm.WriteUInt64(nStartRow);
+        mrStrm.WriteUInt64(nDataSize);
+        switch (node.type)
+        {
+            case sc::element_type_empty:
+            {
+                mrStrm.WriteUChar(0);
+            }
+            break;
+            case sc::element_type_numeric:
+            {
+                mrStrm.WriteUChar(1);
+                sc::numeric_block::const_iterator it = sc::numeric_block::begin(*node.data);
+                std::advance(it, nOffset);
+                sc::numeric_block::const_iterator itEnd = it;
+                std::advance(itEnd, nDataSize);
+
+                for (; it != itEnd; ++it)
+                {
+                    mrStrm.WriteDouble(*it);
+                }
+            }
+            break;
+            case sc::element_type_string:
+            {
+                mrStrm.WriteUChar(2);
+                sc::string_block::const_iterator it = sc::string_block::begin(*node.data);
+                std::advance(it, nOffset);
+                sc::string_block::const_iterator itEnd = it;
+                std::advance(itEnd, nDataSize);
+
+                for (; it != itEnd; ++it)
+                {
+                    OString aStr = OUStringToOString(it->getString(), RTL_TEXTENCODING_UTF8);
+                    sal_Int32 nStrLength = aStr.getLength();
+                    mrStrm.WriteInt32(nStrLength);
+                    mrStrm.WriteCharPtr(aStr.getStr());
+                }
+            }
+            break;
+            case sc::element_type_formula:
+            {
+                mrStrm.WriteUChar(2);
+            }
+            break;
+        }
+    }
+};
+
+}
+
+void ScColumn::StoreToCache(SvStream& rStrm) const
+{
+    rStrm.WriteUInt64(nCol);
+    SCROW nLastRow = GetLastDataPos();
+    rStrm.WriteUInt64(nLastRow + 1); // the rows are zero based
+
+    StoreToCacheFunc aFunc(rStrm);
+    sc::ParseBlock(maCells.begin(), maCells, aFunc, (SCROW)0, nLastRow);
+}
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */

sc/source/core/data/document10.cxx

@@ -939,4 +939,13 @@ sc::ExternalDataMapper& ScDocument::GetExternalDataMapper()
     return *mpDataMapper;
 }
 
+void ScDocument::StoreTabToCache(SCTAB nTab, SvStream& rStrm) const
+{
+    const ScTable* pTab = FetchTable(nTab);
+    if (!pTab)
+        return;
+
+    pTab->StoreToCache(rStrm);
+}
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */

sc/source/core/data/table7.cxx

@@ -417,4 +417,20 @@ void ScTable::finalizeOutlineImport()
     }
 }
 
+void ScTable::StoreToCache(SvStream& rStrm) const
+{
+    SCCOL nStartCol = 0;
+    SCCOL nEndCol = MAXCOL;
+    SCROW nStartRow = 0;
+    SCROW nEndRow = MAXROW;
+
+    GetDataArea(nStartCol, nStartRow, nEndCol, nEndRow, false, false);
+
+    rStrm.WriteUInt64(nEndCol + 1);
+    for (SCCOL nCol = 0; nCol <= nEndCol; ++nCol)
+    {
+        aCol[nCol].StoreToCache(rStrm);
+    }
+}
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */