pdf: refactor and move encryption init. to a common function

This is needed because we have to separate the init. for both
encryption methods and we have to init both when the password is
entered. Currently we only prepared this, to make this possible
when we introduce the other encryption method.

Change-Id: Id6556ddc6a6218164a93bb689f03d6ec6dbad8b9
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/176454
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice@gmail.com>
Reviewed-by: Miklos Vajna <vmiklos@collabora.com>
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/177873
Tested-by: Jenkins
Reviewed-by: Tomaž Vajngerl <quikee@gmail.com>

filter/source/pdf/impdialog.cxx

@@ -23,6 +23,7 @@
 #include <vcl/errinf.hxx>
 #include <vcl/graphic/GraphicMetadata.hxx>
 #include <vcl/svapp.hxx>
+#include <vcl/pdf/PDFEncryptionInitialization.hxx>
 #include <vcl/weld.hxx>
 #include <sfx2/passwd.hxx>
 #include <comphelper/diagnose_ex.hxx>
@@ -1418,7 +1419,7 @@ IMPL_LINK_NOARG(ImpPDFTabSecurityPage, ClickmaPbSetPwdHdl, weld::Button&, void)
             mbHaveUserPassword = !aUserPW.isEmpty();
             mbHaveOwnerPassword = !aOwnerPW.isEmpty();
 
-            mxPreparedPasswords = vcl::PDFWriter::InitEncryption( aOwnerPW, aUserPW );
+            mxPreparedPasswords = vcl::pdf::initEncryption(aOwnerPW, aUserPW);
             if (!mxPreparedPasswords.is())
             {
                 OUString msg;

filter/source/pdf/pdfexport.cxx

@@ -28,6 +28,7 @@
 #include <vcl/mapmod.hxx>
 #include <vcl/gdimtf.hxx>
 #include <vcl/graphic/GraphicMetadata.hxx>
+#include <vcl/pdf/PDFEncryptionInitialization.hxx>
 #include <rtl/ustring.hxx>
 #include <comphelper/propertyvalue.hxx>
 #include <comphelper/sequence.hxx>
@@ -931,7 +932,7 @@ bool PDFExport::Export( const OUString& rFile, const Sequence< PropertyValue >&
                 aContext.Encryption.CanCopyOrExtract                = bCanCopyOrExtract;
                 aContext.Encryption.CanExtractForAccessibility  = bCanExtractForAccessibility;
                 if( bEncrypt && ! xEnc.is() )
-                    xEnc = vcl::PDFWriter::InitEncryption( aPermissionPassword, aOpenPassword );
+                    xEnc = vcl::pdf::initEncryption(aPermissionPassword, aOpenPassword);
                 if( bEncrypt && !aPermissionPassword.isEmpty() && ! aPreparedPermissionPassword.hasElements() )
                     aPreparedPermissionPassword = comphelper::OStorageHelper::CreatePackageEncryptionData( aPermissionPassword );
             }

include/vcl/pdf/PDFEncryptionInitialization.hxx

@@ -0,0 +1,31 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ */
+
+#pragma once
+
+#include <rtl/ustring.hxx>
+#include <vcl/dllapi.h>
+
+namespace com::sun::star::beans
+{
+class XMaterialHolder;
+}
+namespace com::sun::star::uno
+{
+template <typename> class Reference;
+}
+
+namespace vcl::pdf
+{
+VCL_DLLPUBLIC css::uno::Reference<css::beans::XMaterialHolder>
+initEncryption(const OUString& i_rOwnerPassword, const OUString& i_rUserPassword);
+}
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */

vcl/Library_vcl.mk

@@ -511,6 +511,7 @@ $(eval $(call gb_Library_add_exception_objects,vcl,\
     vcl/source/pdf/EncryptionHashTransporter \
     vcl/source/pdf/ExternalPDFStreams \
     vcl/source/pdf/PDFiumTools \
+    vcl/source/pdf/PDFEncryptionInitialization \
     vcl/source/pdf/PDFEncryptor \
     vcl/source/pdf/PDFEncryptorR6 \
     vcl/source/pdf/PdfConfig \

vcl/inc/pdf/PDFEncryptor.hxx

@@ -20,10 +20,6 @@ namespace vcl
 struct PDFEncryptionProperties;
 }
 
-namespace com::sun::star::beans
-{
-class XMaterialHolder;
-}
 namespace com::sun::star::uno
 {
 template <typename> class Reference;
@@ -31,6 +27,8 @@ template <typename> class Reference;
 
 namespace vcl::pdf
 {
+class EncryptionHashTransporter;
+
 class PDFEncryptor : public IPDFEncryptor
 {
 private:
@@ -60,10 +58,10 @@ public:
     sal_Int32 getKeyLength() override { return m_nKeyLength; }
     sal_Int32 getRC4KeyLength() { return m_nRC4KeyLength; }
 
-    static css::uno::Reference<css::beans::XMaterialHolder>
+    static void initEncryption(EncryptionHashTransporter& rEncryptionHashTransporter,
-    initEncryption(const OUString& i_rOwnerPassword, const OUString& i_rUserPassword);
+                               const OUString& i_rOwnerPassword, const OUString& i_rUserPassword);
 
-    virtual bool prepareEncryption(
+    bool prepareEncryption(
         const css::uno::Reference<css::beans::XMaterialHolder>& xEncryptionMaterialHolder,
         PDFEncryptionProperties& rProperties) override;
 

vcl/source/gdi/pdfwriter.cxx

@@ -20,6 +20,7 @@
 #include <vcl/bitmapex.hxx>
 
 #include <pdf/pdfwriter_impl.hxx>
+#include <vcl/pdf/PDFEncryptionInitialization.hxx>
 
 using namespace vcl;
 
@@ -471,7 +472,7 @@ std::set< PDFWriter::ErrorCode > const & PDFWriter::GetErrors() const
 css::uno::Reference< css::beans::XMaterialHolder >
 PDFWriter::InitEncryption(const OUString& i_rOwnerPassword, const OUString& i_rUserPassword)
 {
-    return PDFEncryptor::initEncryption(i_rOwnerPassword, i_rUserPassword);
+    return vcl::pdf::initEncryption(i_rOwnerPassword, i_rUserPassword);
 }
 
 void PDFWriter::PlayMetafile( const GDIMetaFile& i_rMTF, const vcl::PDFWriter::PlayMetafileContext& i_rPlayContext, PDFExtOutDevData* i_pData )

vcl/source/pdf/PDFEncryptionInitialization.cxx

@@ -0,0 +1,32 @@
+/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+/*
+ * This file is part of the LibreOffice project.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ */
+
+#include <vcl/pdf/PDFEncryptionInitialization.hxx>
+#include <pdf/EncryptionHashTransporter.hxx>
+#include <com/sun/star/beans/XMaterialHolder.hpp>
+#include <rtl/ref.hxx>
+#include <pdf/PDFEncryptor.hxx>
+#include <pdf/PDFEncryptorR6.hxx>
+
+using namespace css;
+
+namespace vcl::pdf
+{
+css::uno::Reference<css::beans::XMaterialHolder> initEncryption(const OUString& i_rOwnerPassword,
+                                                                const OUString& i_rUserPassword)
+{
+    rtl::Reference<EncryptionHashTransporter> pTransporter = new EncryptionHashTransporter;
+    PDFEncryptor::initEncryption(*pTransporter, i_rOwnerPassword, i_rUserPassword);
+    return pTransporter;
+}
+
+} // end vcl::pdf
+
+/* vim:set shiftwidth=4 softtabstop=4 expandtab: */

vcl/source/pdf/PDFEncryptor.cxx

@@ -327,32 +327,26 @@ PDFEncryptor::~PDFEncryptor() { rtl_cipher_destroyARCFOUR(m_aCipher); }
 1. init the document id, used both for building the document id and for building the encryption key(s)
 2. build the encryption key following algorithms described in the PDF specification
  */
-uno::Reference<beans::XMaterialHolder>
+void PDFEncryptor::initEncryption(EncryptionHashTransporter& rEncryptionHashTransporter,
-PDFEncryptor::initEncryption(const OUString& i_rOwnerPassword, const OUString& i_rUserPassword)
+                                  const OUString& i_rOwnerPassword, const OUString& i_rUserPassword)
 {
-    uno::Reference<beans::XMaterialHolder> xResult;
     if (!i_rOwnerPassword.isEmpty() || !i_rUserPassword.isEmpty())
     {
-        rtl::Reference<EncryptionHashTransporter> pTransporter = new EncryptionHashTransporter;
-        xResult = pTransporter;
-
         // get padded passwords
         sal_uInt8 aPadUPW[ENCRYPTED_PWD_SIZE], aPadOPW[ENCRYPTED_PWD_SIZE];
         padPassword(i_rOwnerPassword.isEmpty() ? i_rUserPassword : i_rOwnerPassword, aPadOPW);
         padPassword(i_rUserPassword, aPadUPW);
 
-        if (computeODictionaryValue(aPadOPW, aPadUPW, pTransporter->getOValue(), SECUR_128BIT_KEY))
+        if (computeODictionaryValue(aPadOPW, aPadUPW, rEncryptionHashTransporter.getOValue(),
+                                    SECUR_128BIT_KEY))
         {
-            pTransporter->getUDigest()->update(aPadUPW, ENCRYPTED_PWD_SIZE);
+            rEncryptionHashTransporter.getUDigest()->update(aPadUPW, ENCRYPTED_PWD_SIZE);
         }
-        else
-            xResult.clear();
 
         // trash temporary padded cleartext PWDs
         rtl_secureZeroMemory(aPadOPW, sizeof(aPadOPW));
         rtl_secureZeroMemory(aPadUPW, sizeof(aPadUPW));
     }
-    return xResult;
 }
 
 bool PDFEncryptor::prepareEncryption(