<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: itemholder1.cxx,v $
 *
 *  $Revision: 1.12 $
 *
 *  last change: $Author: rt $ $Date: 2007-07-26 08:44:42 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

// MARKER(update_precomp.py): autogen include statement, do not remove
#include "precompiled_svtools.hxx"

#include "itemholder1.hxx"

//-----------------------------------------------
// includes

#ifndef _COMPHELPER_PROCESSFACTORY_HXX_
#include <comphelper/processfactory.hxx>
#endif

#ifndef _COM_SUN_STAR_LANG_XCOMPONENT_HPP_
#include <com/sun/star/lang/XComponent.hpp>
#endif

#include <svtools/accelcfg.hxx>
#include <svtools/addxmltostorageoptions.hxx>
#include <cacheoptions.hxx>
#include <svtools/cmdoptions.hxx>
#include <svtools/compatibility.hxx>
#include <svtools/defaultoptions.hxx>
#include <svtools/dynamicmenuoptions.hxx>
#include <eventcfg.hxx>
#include <svtools/extendedsecurityoptions.hxx>
#include <fltrcfg.hxx>
#include <svtools/fontoptions.hxx>
#include <svtools/historyoptions.hxx>
#include <svtools/inetoptions.hxx>
#include <svtools/internaloptions.hxx>
#include <javaoptions.hxx>
#include <svtools/lingucfg.hxx>
#include <svtools/localisationoptions.hxx>
#include <svtools/menuoptions.hxx>
#include <svtools/moduleoptions.hxx>
#include <svtools/options3d.hxx>
#include <svtools/pathoptions.hxx>
#include <svtools/printwarningoptions.hxx>
#include <regoptions.hxx>
#include <svtools/optionsdlg.hxx>
#include <svtools/saveopt.hxx>
#include <searchopt.hxx>
#include <svtools/securityoptions.hxx>
#include <svtools/sourceviewconfig.hxx>
#include <svtools/startoptions.hxx>
#include <svtools/viewoptions.hxx>
#include <svtools/workingsetoptions.hxx>
#include <xmlaccelcfg.hxx>
#include <svtools/options.hxx>

//-----------------------------------------------
// namespaces

namespace css = ::com::sun::star;

//-----------------------------------------------
// declarations

//-----------------------------------------------
ItemHolder1::ItemHolder1()
    : ItemHolderMutexBase()
{
    try
    {
        css::uno::Reference< css::lang::XMultiServiceFactory > xSMGR = ::comphelper::getProcessServiceFactory();
        css::uno::Reference< css::lang::XComponent > xCfg(
            xSMGR->createInstance(::rtl::OUString::createFromAscii("com.sun.star.configuration.ConfigurationProvider")),
            css::uno::UNO_QUERY);
        if (xCfg.is())
            xCfg->addEventListener(static_cast< css::lang::XEventListener* >(this));
    }
// #i37892  got errorhandling from   ConfigManager::GetConfigurationProvider()
#ifdef DBG_UTIL
    catch(css::uno::Exception& rEx)
    {
        static sal_Bool bMessage = sal_True;
        if(bMessage)
        {
            bMessage = sal_False;
            ::rtl::OString sMsg("CreateInstance with arguments exception: ");
            sMsg += ::rtl::OString(rEx.Message.getStr(),
                        rEx.Message.getLength(),
                        RTL_TEXTENCODING_ASCII_US);
            OSL_ENSURE(sal_False, sMsg.getStr());
        }
    }
#else
    catch(css::uno::Exception&){}
#endif
}

//-----------------------------------------------
ItemHolder1::~ItemHolder1()
{
    impl_releaseAllItems();
}

//-----------------------------------------------
void ItemHolder1::holdConfigItem(EItem eItem)
{
    static ItemHolder1* pHolder = new ItemHolder1();
    pHolder->impl_addItem(eItem);
}

//-----------------------------------------------
void SAL_CALL ItemHolder1::disposing(const css::lang::EventObject&)
    throw(css::uno::RuntimeException)
{
    css::uno::Reference< css::uno::XInterface > xSelfHold(static_cast< css::lang::XEventListener* >(this), css::uno::UNO_QUERY);
    impl_releaseAllItems();
}

//-----------------------------------------------
void ItemHolder1::impl_addItem(EItem eItem)
{
    ::osl::ResettableMutexGuard aLock(m_aLock);

    TItems::const_iterator pIt;
    for (  pIt  = m_lItems.begin();
           pIt != m_lItems.end()  ;
         ++pIt                    )
    {
        const TItemInfo& rInfo = *pIt;
        if (rInfo.eItem == eItem)
            return;
    }

    TItemInfo aNewItem;
    aNewItem.eItem = eItem;
    impl_newItem(aNewItem);
    if (aNewItem.pItem)
        m_lItems.push_back(aNewItem);
}

//-----------------------------------------------
void ItemHolder1::impl_releaseAllItems()
{
    ::osl::ResettableMutexGuard aLock(m_aLock);

    TItems::iterator pIt;
    for (  pIt  = m_lItems.begin();
           pIt != m_lItems.end()  ;
         ++pIt                    )
    {
        TItemInfo& rInfo = *pIt;
        impl_deleteItem(rInfo);
    }
    m_lItems.clear();
}

//-----------------------------------------------
void ItemHolder1::impl_newItem(TItemInfo& rItem)
{
    switch(rItem.eItem)
    {
        case E_ACCELCFG :
            rItem.pItem = new SvtAcceleratorConfiguration();
            break;

        case E_ADDXMLTOSTORAGEOPTIONS :
            rItem.pItem = new SvtAddXMLToStorageOptions();
            break;

        case E_CMDOPTIONS :
            rItem.pItem = new SvtCommandOptions();
            break;

        case E_COMPATIBILITY :
            rItem.pItem = new SvtCompatibilityOptions();
            break;

        case E_DEFAULTOPTIONS :
            rItem.pItem = new SvtDefaultOptions();
            break;

        case E_DYNAMICMENUOPTIONS :
            rItem.pItem = new SvtDynamicMenuOptions();
            break;

        case E_EVENTCFG :
            //rItem.pItem = new GlobalEventConfig();
            break;

        case E_EXTENDEDSECURITYOPTIONS :
            rItem.pItem = new SvtExtendedSecurityOptions();
            break;

        case E_FLTRCFG :
// no ref count            rItem.pItem = new SvtFilterOptions();
            break;

        case E_FONTOPTIONS :
            rItem.pItem = new SvtFontOptions();
            break;

        case E_HISTORYOPTIONS :
            rItem.pItem = new SvtHistoryOptions();
            break;

        case E_INETOPTIONS :
            rItem.pItem = new SvtInetOptions();
            break;

        case E_INTERNALOPTIONS :
            rItem.pItem = new SvtInternalOptions();
            break;

        case E_JAVAOPTIONS :
// no ref count            rItem.pItem = new SvtJavaOptions();
            break;

        case E_LINGUCFG :
            rItem.pItem = new SvtLinguConfig();
            break;

        case E_LOCALISATIONOPTIONS :
            rItem.pItem = new SvtLocalisationOptions();
            break;

        case E_MENUOPTIONS :
            rItem.pItem = new SvtMenuOptions();
            break;

        case E_MODULEOPTIONS :
            rItem.pItem = new SvtModuleOptions();
            break;

        case E_OPTIONSDLGOPTIONS :
            rItem.pItem = new SvtOptionsDialogOptions();
            break;

        case E_OPTIONS3D :
            rItem.pItem = new SvtOptions3D();
            break;

        case E_PATHOPTIONS :
            rItem.pItem = new SvtPathOptions();
            break;

        case E_PRINTWARNINGOPTIONS :
            rItem.pItem = new SvtPrintWarningOptions();
            break;

        case E_REGOPTIONS :
// no ref count            rItem.pItem = new ::svt::RegOptions();
            break;

        case E_SAVEOPTIONS :
            rItem.pItem = new SvtSaveOptions();
            break;

        case E_SEARCHOPT :
// no ref count            rItem.pItem = new SvtSearchOptions();
            break;

        case E_SECURITYOPTIONS :
            rItem.pItem = new SvtSecurityOptions();
            break;

        case E_SOURCEVIEWCONFIG :
            rItem.pItem = new ::svt::SourceViewConfig();
            break;

        case E_STARTOPTIONS :
            rItem.pItem = new SvtStartOptions();
            break;

        case E_VIEWOPTIONS_DIALOG :
            rItem.pItem = new SvtViewOptions(E_DIALOG, ::rtl::OUString());
            break;

        case E_VIEWOPTIONS_TABDIALOG :
            rItem.pItem = new SvtViewOptions(E_TABDIALOG, ::rtl::OUString());
            break;

        case E_VIEWOPTIONS_TABPAGE :
            rItem.pItem = new SvtViewOptions(E_TABPAGE, ::rtl::OUString());
            break;

        case E_VIEWOPTIONS_WINDOW :
            rItem.pItem = new SvtViewOptions(E_WINDOW, ::rtl::OUString());
            break;

        case E_WORKINGSETOPTIONS :
            rItem.pItem = new SvtWorkingSetOptions();
            break;

        case E_XMLACCELCFG :
            // ??? TODO
            break;
        default:
            OSL_ASSERT( "unknown item type" );
            break;
    }
}

//-----------------------------------------------
void ItemHolder1::impl_deleteItem(TItemInfo& rItem)
{
    if (rItem.pItem)
    {
        delete rItem.pItem;
        rItem.pItem = 0;
    }
}

<commit_msg>INTEGRATION: CWS changefileheader (1.12.210); FILE MERGED 2008/04/01 15:44:47 thb 1.12.210.3: #i85898# Stripping all external header guards 2008/04/01 12:43:24 thb 1.12.210.2: #i85898# Stripping all external header guards 2008/03/31 13:01:25 rt 1.12.210.1: #i87441# Change license header to LPGL v3.<commit_after>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2008 by Sun Microsystems, Inc.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * $RCSfile: itemholder1.cxx,v $
 * $Revision: 1.13 $
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

// MARKER(update_precomp.py): autogen include statement, do not remove
#include "precompiled_svtools.hxx"

#include "itemholder1.hxx"

//-----------------------------------------------
// includes
#include <comphelper/processfactory.hxx>
#include <com/sun/star/lang/XComponent.hpp>

#include <svtools/accelcfg.hxx>
#include <svtools/addxmltostorageoptions.hxx>
#include <cacheoptions.hxx>
#include <svtools/cmdoptions.hxx>
#include <svtools/compatibility.hxx>
#include <svtools/defaultoptions.hxx>
#include <svtools/dynamicmenuoptions.hxx>
#include <eventcfg.hxx>
#include <svtools/extendedsecurityoptions.hxx>
#include <fltrcfg.hxx>
#include <svtools/fontoptions.hxx>
#include <svtools/historyoptions.hxx>
#include <svtools/inetoptions.hxx>
#include <svtools/internaloptions.hxx>
#include <javaoptions.hxx>
#include <svtools/lingucfg.hxx>
#include <svtools/localisationoptions.hxx>
#include <svtools/menuoptions.hxx>
#include <svtools/moduleoptions.hxx>
#include <svtools/options3d.hxx>
#include <svtools/pathoptions.hxx>
#include <svtools/printwarningoptions.hxx>
#include <regoptions.hxx>
#include <svtools/optionsdlg.hxx>
#include <svtools/saveopt.hxx>
#include <searchopt.hxx>
#include <svtools/securityoptions.hxx>
#include <svtools/sourceviewconfig.hxx>
#include <svtools/startoptions.hxx>
#include <svtools/viewoptions.hxx>
#include <svtools/workingsetoptions.hxx>
#include <xmlaccelcfg.hxx>
#include <svtools/options.hxx>

//-----------------------------------------------
// namespaces

namespace css = ::com::sun::star;

//-----------------------------------------------
// declarations

//-----------------------------------------------
ItemHolder1::ItemHolder1()
    : ItemHolderMutexBase()
{
    try
    {
        css::uno::Reference< css::lang::XMultiServiceFactory > xSMGR = ::comphelper::getProcessServiceFactory();
        css::uno::Reference< css::lang::XComponent > xCfg(
            xSMGR->createInstance(::rtl::OUString::createFromAscii("com.sun.star.configuration.ConfigurationProvider")),
            css::uno::UNO_QUERY);
        if (xCfg.is())
            xCfg->addEventListener(static_cast< css::lang::XEventListener* >(this));
    }
// #i37892  got errorhandling from   ConfigManager::GetConfigurationProvider()
#ifdef DBG_UTIL
    catch(css::uno::Exception& rEx)
    {
        static sal_Bool bMessage = sal_True;
        if(bMessage)
        {
            bMessage = sal_False;
            ::rtl::OString sMsg("CreateInstance with arguments exception: ");
            sMsg += ::rtl::OString(rEx.Message.getStr(),
                        rEx.Message.getLength(),
                        RTL_TEXTENCODING_ASCII_US);
            OSL_ENSURE(sal_False, sMsg.getStr());
        }
    }
#else
    catch(css::uno::Exception&){}
#endif
}

//-----------------------------------------------
ItemHolder1::~ItemHolder1()
{
    impl_releaseAllItems();
}

//-----------------------------------------------
void ItemHolder1::holdConfigItem(EItem eItem)
{
    static ItemHolder1* pHolder = new ItemHolder1();
    pHolder->impl_addItem(eItem);
}

//-----------------------------------------------
void SAL_CALL ItemHolder1::disposing(const css::lang::EventObject&)
    throw(css::uno::RuntimeException)
{
    css::uno::Reference< css::uno::XInterface > xSelfHold(static_cast< css::lang::XEventListener* >(this), css::uno::UNO_QUERY);
    impl_releaseAllItems();
}

//-----------------------------------------------
void ItemHolder1::impl_addItem(EItem eItem)
{
    ::osl::ResettableMutexGuard aLock(m_aLock);

    TItems::const_iterator pIt;
    for (  pIt  = m_lItems.begin();
           pIt != m_lItems.end()  ;
         ++pIt                    )
    {
        const TItemInfo& rInfo = *pIt;
        if (rInfo.eItem == eItem)
            return;
    }

    TItemInfo aNewItem;
    aNewItem.eItem = eItem;
    impl_newItem(aNewItem);
    if (aNewItem.pItem)
        m_lItems.push_back(aNewItem);
}

//-----------------------------------------------
void ItemHolder1::impl_releaseAllItems()
{
    ::osl::ResettableMutexGuard aLock(m_aLock);

    TItems::iterator pIt;
    for (  pIt  = m_lItems.begin();
           pIt != m_lItems.end()  ;
         ++pIt                    )
    {
        TItemInfo& rInfo = *pIt;
        impl_deleteItem(rInfo);
    }
    m_lItems.clear();
}

//-----------------------------------------------
void ItemHolder1::impl_newItem(TItemInfo& rItem)
{
    switch(rItem.eItem)
    {
        case E_ACCELCFG :
            rItem.pItem = new SvtAcceleratorConfiguration();
            break;

        case E_ADDXMLTOSTORAGEOPTIONS :
            rItem.pItem = new SvtAddXMLToStorageOptions();
            break;

        case E_CMDOPTIONS :
            rItem.pItem = new SvtCommandOptions();
            break;

        case E_COMPATIBILITY :
            rItem.pItem = new SvtCompatibilityOptions();
            break;

        case E_DEFAULTOPTIONS :
            rItem.pItem = new SvtDefaultOptions();
            break;

        case E_DYNAMICMENUOPTIONS :
            rItem.pItem = new SvtDynamicMenuOptions();
            break;

        case E_EVENTCFG :
            //rItem.pItem = new GlobalEventConfig();
            break;

        case E_EXTENDEDSECURITYOPTIONS :
            rItem.pItem = new SvtExtendedSecurityOptions();
            break;

        case E_FLTRCFG :
// no ref count            rItem.pItem = new SvtFilterOptions();
            break;

        case E_FONTOPTIONS :
            rItem.pItem = new SvtFontOptions();
            break;

        case E_HISTORYOPTIONS :
            rItem.pItem = new SvtHistoryOptions();
            break;

        case E_INETOPTIONS :
            rItem.pItem = new SvtInetOptions();
            break;

        case E_INTERNALOPTIONS :
            rItem.pItem = new SvtInternalOptions();
            break;

        case E_JAVAOPTIONS :
// no ref count            rItem.pItem = new SvtJavaOptions();
            break;

        case E_LINGUCFG :
            rItem.pItem = new SvtLinguConfig();
            break;

        case E_LOCALISATIONOPTIONS :
            rItem.pItem = new SvtLocalisationOptions();
            break;

        case E_MENUOPTIONS :
            rItem.pItem = new SvtMenuOptions();
            break;

        case E_MODULEOPTIONS :
            rItem.pItem = new SvtModuleOptions();
            break;

        case E_OPTIONSDLGOPTIONS :
            rItem.pItem = new SvtOptionsDialogOptions();
            break;

        case E_OPTIONS3D :
            rItem.pItem = new SvtOptions3D();
            break;

        case E_PATHOPTIONS :
            rItem.pItem = new SvtPathOptions();
            break;

        case E_PRINTWARNINGOPTIONS :
            rItem.pItem = new SvtPrintWarningOptions();
            break;

        case E_REGOPTIONS :
// no ref count            rItem.pItem = new ::svt::RegOptions();
            break;

        case E_SAVEOPTIONS :
            rItem.pItem = new SvtSaveOptions();
            break;

        case E_SEARCHOPT :
// no ref count            rItem.pItem = new SvtSearchOptions();
            break;

        case E_SECURITYOPTIONS :
            rItem.pItem = new SvtSecurityOptions();
            break;

        case E_SOURCEVIEWCONFIG :
            rItem.pItem = new ::svt::SourceViewConfig();
            break;

        case E_STARTOPTIONS :
            rItem.pItem = new SvtStartOptions();
            break;

        case E_VIEWOPTIONS_DIALOG :
            rItem.pItem = new SvtViewOptions(E_DIALOG, ::rtl::OUString());
            break;

        case E_VIEWOPTIONS_TABDIALOG :
            rItem.pItem = new SvtViewOptions(E_TABDIALOG, ::rtl::OUString());
            break;

        case E_VIEWOPTIONS_TABPAGE :
            rItem.pItem = new SvtViewOptions(E_TABPAGE, ::rtl::OUString());
            break;

        case E_VIEWOPTIONS_WINDOW :
            rItem.pItem = new SvtViewOptions(E_WINDOW, ::rtl::OUString());
            break;

        case E_WORKINGSETOPTIONS :
            rItem.pItem = new SvtWorkingSetOptions();
            break;

        case E_XMLACCELCFG :
            // ??? TODO
            break;
        default:
            OSL_ASSERT( "unknown item type" );
            break;
    }
}

//-----------------------------------------------
void ItemHolder1::impl_deleteItem(TItemInfo& rItem)
{
    if (rItem.pItem)
    {
        delete rItem.pItem;
        rItem.pItem = 0;
    }
}

<|endoftext|>


<commit_before>
// grindtrick import pstream
// grindtrick import boost_locale
// grindtrick import boost_filesystem
// grindtrick import boost_system

#include <algorithm>
#include <iostream>
#include <iterator>
#include <fstream>
#include <vector>
#include <regex>
#include <set>
#include <boost/range/algorithm/count_if.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/algorithm/string/erase.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/xml_parser.hpp>
#include <boost/locale/encoding.hpp>
#include <boost/filesystem.hpp>

#include <pstream.h>

using namespace boost::property_tree;
using namespace boost::locale::conv;
using namespace boost::algorithm;
using namespace boost::range;
using namespace boost::filesystem;
using namespace std;

#include "base64.cpp"

wstring g_sphere_tag;

wstring const SUBJECT_FIELD_NAME(L"Sujet");
wstring const TAG_FIELD_NAME(L"\u00C9tiquettes");

class check_error
{
public:
    check_error(wstring const & msg) : msg_(msg) {}

    wstring msg_;

    wstring user_message() const
    {
        return msg_;
    }
};

wstring to_markdown(wstring const & content)
{
    //wofstream ifs("to_markdown_input");
    //ifs << content;

    using namespace redi;

    vector<string> args;
    args.push_back("node");
    args.push_back("to_markdown.js");

    pstream in("node", args, pstreambuf::pstdin | pstreambuf::pstdout);

    string utf8 = utf_to_utf<char, wchar_t>(content.c_str());

    in << utf8;

    peof(in); // close stdin so client knows to finish

    //ofstream of("to_markdown_result");

    string line;
    string markdown;
    while( getline(in, line) )
    {
        markdown = markdown + line + "\n";
        //of << line << '\n';
    }

    auto r = utf_to_utf<wchar_t, char>(markdown.c_str());

    replace_all(r, L" ", L" ");

    return r;
}

class Attachment
{
public:
    wstring data_;
    wstring encoding_;
    wstring filename_;
};

class Note
{
public:
    wstring title_;
    wstring content_;
    set<wstring> tags_;
    wstring project_;
    vector<Attachment> attachments_;
};

Attachment resource(wptree const & pt)
{
    Attachment a;

    auto data_node = pt.get_child(L"data");

    a.data_ = data_node.get_value<wstring>();

    for(auto v: data_node)
    {
        if( v.first == L"<xmlattr>" )
        {
            for(auto v: v.second)
            {
                if( v.first == L"encoding" )
                {
                    a.encoding_ = v.second.get_value<wstring>();
                }
            }
        }
    }

    auto opt_res_attr = pt.get_child_optional(L"resource-attributes");
    if( opt_res_attr )
    {
        auto res_attr = *opt_res_attr;
        auto opt_fn = res_attr.get_optional<wstring>(L"file-name");
        if( opt_fn ) a.filename_ = *opt_fn;
    }

    return a;
}

bool is_valid_fn_char(wchar_t c)
{
    if( c < 32 ) return false;
    wstring invalids = L"<>:\"/\\|?*.";
    if( count(invalids.begin(), invalids.end(), c) != 0 ) return false;
    return true;
}

void check_for_valid_tag(wstring const & t)
{
    if( t.empty() ) throw check_error(L"tag cannot be empty string");
    if( count( t.begin(), t.end(), L'#' ) != 0 ) throw check_error(L"Evernote tag cannot contain hash");
    if( count_if( t, iswspace ) ) throw check_error(L"tag must be a single word");
    if( ! all_of(t.begin(), t.end(), is_valid_fn_char) ) throw check_error(L"tag contains invalid characters");
}

void repair_title(wstring & t)
{
    wstring r;
    copy_if(t.begin(), t.end(), back_inserter(r), is_valid_fn_char);
    t = r;
    trim(t);
}

void check_for_valid_title(wstring const & t)
{
    if( ! all_of(t.begin(), t.end(), is_valid_fn_char) ) throw check_error(L"title contains invalid characters");
}

void write_attachment(boost::filesystem::wpath const & annex_path, Attachment const & a, int & counter)
{
    auto afn = a.filename_;

    if( afn.empty() )
    {
        wostringstream afn_os;
        afn_os << setw(5) << setfill(L'0')  << counter;
        ++ counter;
        afn = afn_os.str();
    }

    boost::filesystem::wpath path = annex_path / afn;

    if( a.encoding_ != L"base64" )
    {
        throw check_error(L"unsupported attachment encoding: " + a.encoding_);
    }

    string data;
    data.resize(a.data_.size());

    /*
    Note: copy wchar_t down to char.  This is OK
    because the string is base64 encoded data, so
    no character is beyond 127.
    */
    copy(a.data_.begin(), a.data_.end(), data.begin());

    erase_all(data, "\n");
    erase_all(data, "\r");

    auto dec = b64decode(data);
    auto fn = path.string<string>();
    std::ofstream os(fn, ios::binary);
    os.exceptions(~ios::goodbit);
    os << dec;
}

void write_note(Note const & n)
{
    wstring wfn = g_sphere_tag + L" " + n.project_ + L" " + n.title_ + L".md";
    string fn = from_utf(wfn, "UTF-8") ;

    std::wofstream os(fn);

    os << SUBJECT_FIELD_NAME << ": " << n.title_ << '\n';

    os << TAG_FIELD_NAME << ": ";

    for(auto t: n.tags_)
    {
        os << L'#' << t << L' ';
    }
    os << '\n';

    os << '\n';

    os << n.content_;

    if( !n.attachments_.empty() )
    {
        boost::filesystem::wpath note_fn(wfn);
        boost::filesystem::wpath annex_fn = note_fn.stem().wstring() + L".annexes";
        create_directory(annex_fn);
        int counter = 1;
        for(auto a: n.attachments_)
        {
            write_attachment(annex_fn, a, counter);
        }
    }
}

void note(wptree const & pt)
{
    Note n;

    for(auto v: pt)
    {
        if( v.first == L"title" )
        {
            n.title_ = v.second.get_value<wstring>();
            trim(n.title_);
            repair_title(n.title_);
            check_for_valid_title(n.title_);
        }
        else if( v.first == L"content")
        {
            wstring en_note = v.second.get_value<wstring>();
            // en_note is a full XML document.
            // The HTML of the note is inside the <en-note> tag.

            wptree en_note_ptree;
            wistringstream is(en_note);
            read_xml(is, en_note_ptree);

            auto enote = en_note_ptree.get_child(L"en-note");

            wostringstream os;
            write_xml(os, enote);

            n.content_ = to_markdown( os.str() );

            // TODO: sometimes, there is a surrounding <pre>
            // tag.  Useless.  Please remove.
        }
        else if( v.first == L"tag")
        {
            auto tag = v.second.get_value<wstring>();
            trim(tag);
            check_for_valid_tag(tag);
            n.tags_.insert(tag);
        }
        else if( v.first == L"resource")
        {
            n.attachments_.push_back( resource(v.second) );
        }
    }

    n.tags_.insert(g_sphere_tag);

    n.project_ = L"imported";
    for(auto v: n.tags_)
    {
        if( v != g_sphere_tag )
        {
            n.project_ = v;
            break;
        }
    }

    n.tags_.insert(L"imported");

    write_note(n);
}

int main(int argc, char ** argv)
{
    // Load the system locale.
    setlocale(LC_ALL, "");

    try
    {
        g_sphere_tag = L"inro";

        if( argc >= 2 )
        {
            g_sphere_tag = to_utf<wchar_t>(argv[1], "UTF-8");
        }

        regex wc("[0-9]*\\.xml");
        smatch mo;

        for(auto e: directory_iterator("."))
        {
            if( !is_regular_file(e) ) continue;

            string fn = e.path().filename().string();

            if( !regex_match(fn, mo, wc) ) continue;

            std::wifstream note_file(e.path().string());
            note_file.exceptions(~ios::goodbit);

            wptree pt;
            read_xml(note_file, pt);

            cout << fn << '\n';

            try
            {
                note(pt);
                remove(e.path());
            }
            catch(exception const & ex)
            {
                cerr << "exception: " << ex.what() << '\n';
            }
            catch(check_error const & ex)
            {
                wcerr << fn << ": " << ex.user_message() << '\n';
            }
        }
    }
    catch(exception const & ex)
    {
        cerr << "exception: " << ex.what() << '\n';
        return 1;
    }
    catch(check_error const & ex)
    {
        wcerr << ex.user_message() << '\n';
        return 1;
    }

    return 0;
}
<commit_msg>Remove <pre> tags.<commit_after>
// grindtrick import pstream
// grindtrick import boost_locale
// grindtrick import boost_filesystem
// grindtrick import boost_system

#include <algorithm>
#include <iostream>
#include <iterator>
#include <fstream>
#include <vector>
#include <regex>
#include <set>
#include <boost/range/algorithm/count_if.hpp>
#include <boost/algorithm/string/replace.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <boost/algorithm/string/erase.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/xml_parser.hpp>
#include <boost/locale/encoding.hpp>
#include <boost/filesystem.hpp>

#include <pstream.h>

using namespace boost::property_tree;
using namespace boost::locale::conv;
using namespace boost::algorithm;
using namespace boost::range;
using namespace boost::filesystem;
using namespace std;

#include "base64.cpp"

wstring g_sphere_tag;

wstring const SUBJECT_FIELD_NAME(L"Sujet");
wstring const TAG_FIELD_NAME(L"\u00C9tiquettes");

class check_error
{
public:
    check_error(wstring const & msg) : msg_(msg) {}

    wstring msg_;

    wstring user_message() const
    {
        return msg_;
    }
};

wstring to_markdown(wstring const & content)
{
    //wofstream ifs("to_markdown_input");
    //ifs << content;

    using namespace redi;

    vector<string> args;
    args.push_back("node");
    args.push_back("to_markdown.js");

    pstream in("node", args, pstreambuf::pstdin | pstreambuf::pstdout);

    string utf8 = utf_to_utf<char, wchar_t>(content.c_str());

    in << utf8;

    peof(in); // close stdin so client knows to finish

    //ofstream of("to_markdown_result");

    string line;
    string markdown;
    while( getline(in, line) )
    {
        markdown = markdown + line + "\n";
        //of << line << '\n';
    }

    auto r = utf_to_utf<wchar_t, char>(markdown.c_str());

    replace_all(r, L" ", L" ");

    return r;
}

class Attachment
{
public:
    wstring data_;
    wstring encoding_;
    wstring filename_;
};

class Note
{
public:
    wstring title_;
    wstring content_;
    set<wstring> tags_;
    wstring project_;
    vector<Attachment> attachments_;
};

Attachment resource(wptree const & pt)
{
    Attachment a;

    auto data_node = pt.get_child(L"data");

    a.data_ = data_node.get_value<wstring>();

    for(auto v: data_node)
    {
        if( v.first == L"<xmlattr>" )
        {
            for(auto v: v.second)
            {
                if( v.first == L"encoding" )
                {
                    a.encoding_ = v.second.get_value<wstring>();
                }
            }
        }
    }

    auto opt_res_attr = pt.get_child_optional(L"resource-attributes");
    if( opt_res_attr )
    {
        auto res_attr = *opt_res_attr;
        auto opt_fn = res_attr.get_optional<wstring>(L"file-name");
        if( opt_fn ) a.filename_ = *opt_fn;
    }

    return a;
}

bool is_valid_fn_char(wchar_t c)
{
    if( c < 32 ) return false;
    wstring invalids = L"<>:\"/\\|?*.";
    if( count(invalids.begin(), invalids.end(), c) != 0 ) return false;
    return true;
}

void check_for_valid_tag(wstring const & t)
{
    if( t.empty() ) throw check_error(L"tag cannot be empty string");
    if( count( t.begin(), t.end(), L'#' ) != 0 ) throw check_error(L"Evernote tag cannot contain hash");
    if( count_if( t, iswspace ) ) throw check_error(L"tag must be a single word");
    if( ! all_of(t.begin(), t.end(), is_valid_fn_char) ) throw check_error(L"tag contains invalid characters");
}

void repair_title(wstring & t)
{
    wstring r;
    copy_if(t.begin(), t.end(), back_inserter(r), is_valid_fn_char);
    t = r;
    trim(t);
}

void check_for_valid_title(wstring const & t)
{
    if( ! all_of(t.begin(), t.end(), is_valid_fn_char) ) throw check_error(L"title contains invalid characters");
}

void write_attachment(boost::filesystem::wpath const & annex_path, Attachment const & a, int & counter)
{
    auto afn = a.filename_;

    if( afn.empty() )
    {
        wostringstream afn_os;
        afn_os << setw(5) << setfill(L'0')  << counter;
        ++ counter;
        afn = afn_os.str();
    }

    boost::filesystem::wpath path = annex_path / afn;

    if( a.encoding_ != L"base64" )
    {
        throw check_error(L"unsupported attachment encoding: " + a.encoding_);
    }

    string data;
    data.resize(a.data_.size());

    /*
    Note: copy wchar_t down to char.  This is OK
    because the string is base64 encoded data, so
    no character is beyond 127.
    */
    copy(a.data_.begin(), a.data_.end(), data.begin());

    erase_all(data, "\n");
    erase_all(data, "\r");

    auto dec = b64decode(data);
    auto fn = path.string<string>();
    std::ofstream os(fn, ios::binary);
    os.exceptions(~ios::goodbit);
    os << dec;
}

void write_note(Note const & n)
{
    wstring wfn = g_sphere_tag + L" " + n.project_ + L" " + n.title_ + L".md";
    string fn = from_utf(wfn, "UTF-8") ;

    std::wofstream os(fn);

    os << SUBJECT_FIELD_NAME << ": " << n.title_ << '\n';

    os << TAG_FIELD_NAME << ": ";

    for(auto t: n.tags_)
    {
        os << L'#' << t << L' ';
    }
    os << '\n';

    os << '\n';

    os << n.content_;

    if( !n.attachments_.empty() )
    {
        boost::filesystem::wpath note_fn(wfn);
        boost::filesystem::wpath annex_fn = note_fn.stem().wstring() + L".annexes";
        create_directory(annex_fn);
        int counter = 1;
        for(auto a: n.attachments_)
        {
            write_attachment(annex_fn, a, counter);
        }
    }
}

void note(wptree const & pt)
{
    Note n;

    for(auto v: pt)
    {
        if( v.first == L"title" )
        {
            n.title_ = v.second.get_value<wstring>();
            trim(n.title_);
            repair_title(n.title_);
            check_for_valid_title(n.title_);
        }
        else if( v.first == L"content")
        {
            wstring en_note = v.second.get_value<wstring>();
            // en_note is a full XML document.
            // The HTML of the note is inside the <en-note> tag.

            wptree en_note_ptree;
            wistringstream is(en_note);
            read_xml(is, en_note_ptree);

            auto enote = en_note_ptree.get_child(L"en-note");

            auto pre_opt = enote.get_child_optional(L"pre");
            if( pre_opt )
            {
                n.content_ = (*pre_opt).get_value<wstring>();
            }
            else
            {
                wostringstream os;
                write_xml(os, enote);

                n.content_ = to_markdown( os.str() );
            }
        }
        else if( v.first == L"tag")
        {
            auto tag = v.second.get_value<wstring>();
            trim(tag);
            check_for_valid_tag(tag);
            n.tags_.insert(tag);
        }
        else if( v.first == L"resource")
        {
            n.attachments_.push_back( resource(v.second) );
        }
    }

    n.tags_.insert(g_sphere_tag);

    n.project_ = L"imported";
    for(auto v: n.tags_)
    {
        if( v != g_sphere_tag )
        {
            n.project_ = v;
            break;
        }
    }

    n.tags_.insert(L"imported");

    write_note(n);
}

int main(int argc, char ** argv)
{
    // Load the system locale.
    setlocale(LC_ALL, "");

    try
    {
        g_sphere_tag = L"inro";

        if( argc >= 2 )
        {
            g_sphere_tag = to_utf<wchar_t>(argv[1], "UTF-8");
        }

        regex wc("[0-9]*\\.xml");
        smatch mo;

        for(auto e: directory_iterator("."))
        {
            if( !is_regular_file(e) ) continue;

            string fn = e.path().filename().string();

            if( !regex_match(fn, mo, wc) ) continue;

            std::wifstream note_file(e.path().string());
            note_file.exceptions(~ios::goodbit);

            wptree pt;
            read_xml(note_file, pt);

            cout << fn << '\n';

            try
            {
                note(pt);
                remove(e.path());
            }
            catch(exception const & ex)
            {
                cerr << "exception: " << ex.what() << '\n';
            }
            catch(check_error const & ex)
            {
                wcerr << fn << ": " << ex.user_message() << '\n';
            }
        }
    }
    catch(exception const & ex)
    {
        cerr << "exception: " << ex.what() << '\n';
        return 1;
    }
    catch(check_error const & ex)
    {
        wcerr << ex.user_message() << '\n';
        return 1;
    }

    return 0;
}
<|endoftext|>


<commit_before>/**
 *  @copyright Copyright 2016 The J-PET Framework Authors. All rights reserved.
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may find a copy of the License in the LICENCE file.
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *  @file main.cpp
 */

#include <DBHandler/HeaderFiles/DBHandler.h>
#include <JPetManager/JPetManager.h>
#include <JPetTaskLoader/JPetTaskLoader.h>
#include "TaskA.h"
#include "TimeCalibLoader.h"
#include "SignalFinder.h"
#include "HitFinder.h"
#include "TaskC2.h"
#include "TaskD.h"
#include "TaskE.h"

using namespace std;

int main(int argc, char* argv[])
{
  DB::SERVICES::DBHandler::createDBConnection("../DBConfig/configDB.cfg");
  JPetManager& manager = JPetManager::getManager();
  manager.parseCmdLine(argc, argv);

  // Here create all analysis modules to be used:

  manager.registerTask([]() {
    return new JPetTaskLoader("hld",
                              "tslot.raw",
                              new TaskA("TaskA: Unp to Timewindow",
                                        "Process unpacked HLD file into a tree of JPetTimeWindow objects")
                             );
  });

  manager.registerTask([]() {
    return new JPetTaskLoader("tslot.raw", "tslot_calib.raw",
                              new TimeCalibLoader("Apply time corrections from prepared calibrations",
                                  "Apply time corrections from prepared calibrations"));
  });
  manager.registerTask([]() {
    return new JPetTaskLoader("tslot_calib.raw",
                              "raw.sig",
                              new SignalFinder("SignalFinder: Create Raw Sigs",
                                  "Create Raw Signals, optional  draw TOTs per THR",
                                  true)
                             );
  });

  manager.registerTask([]() {
    return new JPetTaskLoader("raw.sig",
                              "phys.sig",
                              new TaskC2("SignalPhysTransform: Create Phys Sigs",
                                  "Create Phys Signals, ")
                             );
  });

  //manager.registerTask([]() {
  //return new JPetTaskLoader("phys_sig", "hits",
  //new HitFinder("Module: Pair signals",
  //"Create hits from physical signals from both ends of scintilators"));
  //});

  // manager.registerTask([](){
  //     return new JPetTaskLoader("phys.hit", "phys.hit.means",
  // 				new TaskD("Module D: Make histograms for hits",
  // 					  "Only make timeDiff histos and produce mean timeDiff value for each threshold and slot to be used by the next module"));
  //   });

  // manager.registerTask([](){
  //     return new JPetTaskLoader("phys.hit.means", "phys.hit.coincplots",
  // 				new TaskE("Module E: Filter hits",
  // 					  "Pass only hits with time diffrerence close to the peak"));
  //   });

  manager.run();
}
<commit_msg>Activate taks chain till hit findiing in main.cpp<commit_after>/**
 *  @copyright Copyright 2016 The J-PET Framework Authors. All rights reserved.
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
 *  You may find a copy of the License in the LICENCE file.
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *  @file main.cpp
 */

#include <DBHandler/HeaderFiles/DBHandler.h>
#include <JPetManager/JPetManager.h>
#include <JPetTaskLoader/JPetTaskLoader.h>
#include "TaskA.h"
#include "TimeCalibLoader.h"
#include "SignalFinder.h"
#include "HitFinder.h"
#include "TaskC2.h"
#include "TaskD.h"
#include "TaskE.h"

using namespace std;

int main(int argc, char* argv[])
{
  //DB::SERVICES::DBHandler::createDBConnection("../DBConfig/configDB.cfg");
  JPetManager& manager = JPetManager::getManager();
  manager.parseCmdLine(argc, argv);

  // Here create all analysis modules to be used:

  manager.registerTask([]() {
    return new JPetTaskLoader("hld",
                              "tslot.raw",
                              new TaskA("TaskA: Unp to Timewindow",
                                        "Process unpacked HLD file into a tree of JPetTimeWindow objects")
                             );
  });

  manager.registerTask([]() {
    return new JPetTaskLoader("tslot.raw", "tslot_calib.raw",
                              new TimeCalibLoader("Apply time corrections from prepared calibrations",
                                  "Apply time corrections from prepared calibrations"));
  });
  manager.registerTask([]() {
    return new JPetTaskLoader("tslot_calib.raw",
                              "raw.sig",
                              new SignalFinder("SignalFinder: Create Raw Sigs",
                                  "Create Raw Signals, optional  draw TOTs per THR",
                                  true)
                             );
  });

  manager.registerTask([]() {
    return new JPetTaskLoader("raw.sig",
                              "phys.sig",
                              new TaskC2("SignalPhysTransform: Create Phys Sigs",
                                         "Create Phys Signals, ")
                             );
  });

  manager.registerTask([]() {
    return new JPetTaskLoader("phys.sig", "hits",
                              new HitFinder("Module: Pair signals",
                                            "Create hits from physical signals from both ends of scintilators"));
  });

  // manager.registerTask([](){
  //     return new JPetTaskLoader("phys.hit", "phys.hit.means",
  // 				new TaskD("Module D: Make histograms for hits",
  // 					  "Only make timeDiff histos and produce mean timeDiff value for each threshold and slot to be used by the next module"));
  //   });

  // manager.registerTask([](){
  //     return new JPetTaskLoader("phys.hit.means", "phys.hit.coincplots",
  // 				new TaskE("Module E: Filter hits",
  // 					  "Pass only hits with time diffrerence close to the peak"));
  //   });

  manager.run();
}
<|endoftext|>


<commit_before>#include <iostream>
#include <fstream>
#include <exception>

#include "Record.h"
#include "Constants.h"
#include "JobOutputStream.h"
#include "SplitJobOutputStream.h"

using namespace std;

class InvalidScanFileException:public exception{
public:
	InvalidScanFileException(string s){message=s;}
	~InvalidScanFileException() throw() {}
	 const char* what() const throw() {return message.c_str();}
private:
		string message;
};

int main(int argc, char** argv)
{
	string freshScanPath="";
	string staleScanPath="";
	string outputDirPath="";

	//get command line args
	for(int i=1; i<argc; i++)
	{
		if(i<argc-1){
			if(string(argv[i])=="-n"){
				i++;
				freshScanPath = argv[i];
			}
			else if(string(argv[i])=="-o"){
				i++;
				staleScanPath = argv[i];
			}
			else if(string(argv[i])=="-f"){
				i++;
				outputDirPath = argv[i];
			}
			else{
				cerr<<"Invalid Option:"<<argv[i]<<endl;
				return 1;
			}

		}
	}

	//error checking
	if(freshScanPath==""||staleScanPath==""||outputDirPath==""){
		if(freshScanPath==""){
			cerr<<"Specify up-to-date scan with -n [file]"<<endl;
		}
		if(staleScanPath==""){
			cerr<<"Specify out-of-date scan with -o [file]"<<endl;
		}
		if(outputDirPath==""){
			cerr<<"Specify output directory with -f [directory]"<<endl;
		}
		return 1;
	}

	//instantiate all objects
	fstream freshScan(freshScanPath.c_str(),ios::in);
	fstream staleScan(staleScanPath.c_str(),ios::in);

	//error checking
	if(!freshScan.good() || !staleScan.good())
	{
		if(!staleScan.good()){
			cerr<<"File path:"<<staleScanPath<<" is inaccessable."<<endl;
		}
		if(!freshScan.good()){
			cerr<<"File path:"<<freshScanPath<<" is inaccessable."<<endl;
		}
		return 2;
	}

	try{
		JobOutputStream createDirs("C",outputDirPath);
		JobOutputStream removeDirs("R",outputDirPath);
		SplitJobOutputStream createFiles("A",outputDirPath);
		SplitJobOutputStream removeFiles("D",outputDirPath);
		SplitJobOutputStream modifyFiles("M",outputDirPath);
		SplitJobOutputStream modifyDirs("Y",outputDirPath);
		JobOutputStream buildLinks("L",outputDirPath);

		Record freshRec, staleRec;
		string tmp;

		while(freshScan && staleScan){
			if(freshRec.isValid() && staleRec.isValid()){

				if(freshRec==staleRec){

					if(freshRec.getType() == Record::Link){
						buildLinks.writeRecord(freshRec);
					}
					else if(freshRec.isModified(staleRec)){
						if(freshRec.getType() == Record::Directory)
							modifyDirs.writeRecord(freshRec);
						else
							modifyFiles.writeRecord(freshRec);
					}

					freshRec.invalidate();
					staleRec.invalidate();

				}
				else if(freshRec < staleRec){
					//record has been added
					if(freshRec.getType() == Record::Directory)
						createDirs.writeRecord(freshRec);
					else if(freshRec.getType() == Record::Link)
						buildLinks.writeRecord(freshRec);
					else
						createFiles.writeRecord(freshRec);
					freshRec.invalidate();
				}
				else{ //staleRec < freshRec
					//record has been removed
					if(staleRec.getType() == Record::Directory)
						removeDirs.writeRecord(staleRec);
					else
						removeFiles.writeRecord(staleRec);
					staleRec.invalidate();
				}
			}

			//get new records
			if(!freshRec.isValid()){
				string oldPath = freshRec[PATH];
				getline(freshScan,tmp);
				freshRec.init(tmp);
				if(oldPath!="" &&freshRec[PATH]<oldPath){
					throw InvalidScanFileException("|"+freshScanPath+":"+freshRec[PATH]+":"+tmp+"|");
				}
			}
			if(!staleRec.isValid()){
				string oldPath = staleRec[PATH];
				getline(staleScan,tmp);
				staleRec.init(tmp);
				if(oldPath!="" &&staleRec[PATH]<oldPath){
					throw InvalidScanFileException("|"+staleScanPath+":"+staleRec[PATH]+":"+tmp+"|");
				}
			}
		}//while both inputs are valid

		//get any remaining fresh records
		while(freshScan){
			if(freshRec.isValid()){
				//record has been added
				if(freshRec.getType() == Record::Directory)
					createDirs.writeRecord(freshRec);
				else if(freshRec.getType() == Record::Link)
					buildLinks.writeRecord(freshRec);
				else
					createFiles.writeRecord(freshRec);
				freshRec.invalidate();
			}

			if(!freshRec.isValid()){
				string oldPath = freshRec[PATH];
				getline(freshScan,tmp);
				freshRec.init(tmp);
				if(oldPath!="" &&freshRec[PATH]<oldPath){
					throw InvalidScanFileException("|"+freshScanPath+":"+freshRec[PATH]+":"+tmp+"|");
				}
			}
		}

		//get any remaining stale records
		while(staleScan){
			if(staleRec.isValid()){
				//record has been removed
				if(staleRec.getType() == Record::Directory)
					removeDirs.writeRecord(staleRec);
				else
					removeFiles.writeRecord(staleRec);
				staleRec.invalidate();
			}
			if(!staleRec.isValid()){
				string oldPath = staleRec[PATH];
				getline(staleScan,tmp);
				staleRec.init(tmp);
				if(oldPath!="" &&staleRec[PATH]<oldPath){
					throw InvalidScanFileException("|"+staleScanPath+":"+staleRec[PATH]+":"+tmp+"|");
				}
			}
		}

	}catch(OutputFailedToOpenException e){
		cerr<<"Output path:"<<outputDirPath<<" is inaccessable."<<endl;
		return 3;
	}catch(IsModRanOnNERecException e){
		cerr<<"One of the scan files is invalid."<<endl;
		return 4;
	}catch(InvalidScanFileException e){
		cerr<<"Scan file:"<<e.what()<<" is not sorted by path."<<endl;
		return 5;
	}catch(exception e){
		cerr<<e.what()<<endl;
		return 999;
	}

	return 0;
}

<commit_msg>better error detection<commit_after>#include <iostream>
#include <fstream>
#include <exception>

#include "Record.h"
#include "Constants.h"
#include "JobOutputStream.h"
#include "SplitJobOutputStream.h"

using namespace std;

class InvalidScanFileException:public exception{
public:
	InvalidScanFileException(string s){message=s;}
	~InvalidScanFileException() throw() {}
	 const char* what() const throw() {return message.c_str();}
private:
		string message;
};

int main(int argc, char** argv)
{
	string freshScanPath="";
	string staleScanPath="";
	string outputDirPath="";

	//get command line args
	for(int i=1; i<argc; i++)
	{
		if(i<argc-1){
			if(string(argv[i])=="-n"){
				i++;
				freshScanPath = argv[i];
			}
			else if(string(argv[i])=="-o"){
				i++;
				staleScanPath = argv[i];
			}
			else if(string(argv[i])=="-f"){
				i++;
				outputDirPath = argv[i];
			}
			else{
				cerr<<"Invalid Option:"<<argv[i]<<endl;
				return 1;
			}

		}
	}

	//error checking
	if(freshScanPath==""||staleScanPath==""||outputDirPath==""){
		if(freshScanPath==""){
			cerr<<"Specify up-to-date scan with -n [file]"<<endl;
		}
		if(staleScanPath==""){
			cerr<<"Specify out-of-date scan with -o [file]"<<endl;
		}
		if(outputDirPath==""){
			cerr<<"Specify output directory with -f [directory]"<<endl;
		}
		return 1;
	}

	//instantiate all objects
	fstream freshScan(freshScanPath.c_str(),ios::in);
	fstream staleScan(staleScanPath.c_str(),ios::in);

	//error checking
	if(!freshScan.good() || !staleScan.good())
	{
		if(!staleScan.good()){
			cerr<<"File path:"<<staleScanPath<<" is inaccessable."<<endl;
		}
		if(!freshScan.good()){
			cerr<<"File path:"<<freshScanPath<<" is inaccessable."<<endl;
		}
		return 2;
	}

	try{
		JobOutputStream createDirs("C",outputDirPath);
		JobOutputStream removeDirs("R",outputDirPath);
		SplitJobOutputStream createFiles("A",outputDirPath);
		SplitJobOutputStream removeFiles("D",outputDirPath);
		SplitJobOutputStream modifyFiles("M",outputDirPath);
		SplitJobOutputStream modifyDirs("Y",outputDirPath);
		JobOutputStream buildLinks("L",outputDirPath);

		Record freshRec, staleRec;
		string tmp;

		while(freshScan && staleScan){
			if(freshRec.isValid() && staleRec.isValid()){

				if(freshRec==staleRec){

					if(freshRec.getType() == Record::Link){
						buildLinks.writeRecord(freshRec);
					}
					else if(freshRec.isModified(staleRec)){
						if(freshRec.getType() == Record::Directory)
							modifyDirs.writeRecord(freshRec);
						else
							modifyFiles.writeRecord(freshRec);
					}

					freshRec.invalidate();
					staleRec.invalidate();

				}
				else if(freshRec < staleRec){
					//record has been added
					if(freshRec.getType() == Record::Directory)
						createDirs.writeRecord(freshRec);
					else if(freshRec.getType() == Record::Link)
						buildLinks.writeRecord(freshRec);
					else
						createFiles.writeRecord(freshRec);
					freshRec.invalidate();
				}
				else{ //staleRec < freshRec
					//record has been removed
					if(staleRec.getType() == Record::Directory)
						removeDirs.writeRecord(staleRec);
					else
						removeFiles.writeRecord(staleRec);
					staleRec.invalidate();
				}
			}

			//get new records
			if(!freshRec.isValid()){
				string oldPath = freshRec[PATH];
				getline(freshScan,tmp);
				freshRec.init(tmp);
				if(oldPath!="" &&freshRec[PATH]<oldPath){
					throw InvalidScanFileException(freshScanPath+"\t"+freshRec[PATH]+"\t"+oldPath);
				}
			}
			if(!staleRec.isValid()){
				string oldPath = staleRec[PATH];
				getline(staleScan,tmp);
				staleRec.init(tmp);
				if(oldPath!="" &&staleRec[PATH]<oldPath){
					throw InvalidScanFileException(staleScanPath+"\t"+staleRec[PATH]+"\t"+oldPath);
				}
			}
		}//while both inputs are valid

		//get any remaining fresh records
		while(freshScan){
			if(freshRec.isValid()){
				//record has been added
				if(freshRec.getType() == Record::Directory)
					createDirs.writeRecord(freshRec);
				else if(freshRec.getType() == Record::Link)
					buildLinks.writeRecord(freshRec);
				else
					createFiles.writeRecord(freshRec);
				freshRec.invalidate();
			}

			if(!freshRec.isValid()){
				string oldPath = freshRec[PATH];
				getline(freshScan,tmp);
				freshRec.init(tmp);
				if(oldPath!="" &&freshRec[PATH]<oldPath){
					throw InvalidScanFileException(freshScanPath+"\t"+freshRec[PATH]+"\t"+oldPath);
				}
			}
		}

		//get any remaining stale records
		while(staleScan){
			if(staleRec.isValid()){
				//record has been removed
				if(staleRec.getType() == Record::Directory)
					removeDirs.writeRecord(staleRec);
				else
					removeFiles.writeRecord(staleRec);
				staleRec.invalidate();
			}
			if(!staleRec.isValid()){
				string oldPath = staleRec[PATH];
				getline(staleScan,tmp);
				staleRec.init(tmp);
				if(oldPath!="" &&staleRec[PATH]<oldPath){
					throw InvalidScanFileException(staleScanPath+"\t"+staleRec[PATH]+"\t"+oldPath);
				}
			}
		}

	}catch(OutputFailedToOpenException e){
		cerr<<"Output path:"<<outputDirPath<<" is inaccessable."<<endl;
		return 3;
	}catch(IsModRanOnNERecException e){
		cerr<<"One of the scan files is invalid."<<endl;
		return 4;
	}catch(InvalidScanFileException e){
		cerr<<"Scan file:"<<e.what()<<" is not sorted by path."<<endl;
		return 5;
	}catch(exception e){
		cerr<<e.what()<<endl;
		return 999;
	}

	return 0;
}

<|endoftext|>


<commit_before>/*
 * Copyright (C) 2018 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "multishard_mutation_query.hh"
#include "schema_registry.hh"
#include "tests/cql_test_env.hh"
#include "tests/eventually.hh"
#include "tests/cql_assertions.hh"
#include "tests/mutation_assertions.hh"
#include "tests/test_table.hh"

#include <seastar/testing/thread_test_case.hh>

#include <experimental/source_location>

const sstring KEYSPACE_NAME = "multishard_mutation_query_test";

static uint64_t aggregate_querier_cache_stat(distributed<database>& db, uint64_t query::querier_cache::stats::*stat) {
    return map_reduce(boost::irange(0u, smp::count), [stat, &db] (unsigned shard) {
        return db.invoke_on(shard, [stat] (database& local_db) {
            auto& stats = local_db.get_querier_cache_stats();
            return stats.*stat;
        });
    }, 0, std::plus<size_t>()).get0();
}

static void check_cache_population(distributed<database>& db, size_t queriers,
        std::experimental::source_location sl = std::experimental::source_location::current()) {
    BOOST_TEST_MESSAGE(format("{}() called from {}() {}:{:d}", __FUNCTION__, sl.function_name(), sl.file_name(), sl.line()));

    parallel_for_each(boost::irange(0u, smp::count), [queriers, &db] (unsigned shard) {
        return db.invoke_on(shard, [queriers] (database& local_db) {
            auto& stats = local_db.get_querier_cache_stats();
            BOOST_REQUIRE_EQUAL(stats.population, queriers);
        });
    }).get0();
}

static void require_eventually_empty_caches(distributed<database>& db,
        std::experimental::source_location sl = std::experimental::source_location::current()) {
    BOOST_TEST_MESSAGE(format("{}() called from {}() {}:{:d}", __FUNCTION__, sl.function_name(), sl.file_name(), sl.line()));

    auto aggregated_population_is_zero = [&] () mutable {
        return aggregate_querier_cache_stat(db, &query::querier_cache::stats::population) == 0;
    };
    BOOST_REQUIRE(eventually_true(aggregated_population_is_zero));
}

// Best run with SMP>=2
SEASTAR_THREAD_TEST_CASE(test_abandoned_read) {
    do_with_cql_env([] (cql_test_env& env) -> future<> {
        using namespace std::chrono_literals;

        env.db().invoke_on_all([] (database& db) {
            db.set_querier_cache_entry_ttl(2s);
        }).get();

        auto [s, _] = test::create_test_table(env, KEYSPACE_NAME, "test_abandoned_read");
        (void)_;

        auto cmd = query::read_command(s->id(), s->version(), s->full_slice(), 7, gc_clock::now(), std::nullopt, query::max_partitions,
                utils::make_random_uuid(), true);

        query_mutations_on_all_shards(env.db(), s, cmd, {query::full_partition_range}, nullptr, std::numeric_limits<uint64_t>::max()).get();

        check_cache_population(env.db(), 1);

        sleep(2s).get();

        require_eventually_empty_caches(env.db());

        return make_ready_future<>();
    }).get();
}

static std::vector<mutation> read_all_partitions_one_by_one(distributed<database>& db, schema_ptr s, std::vector<dht::decorated_key> pkeys) {
    const auto& partitioner = dht::global_partitioner();
    std::vector<mutation> results;
    results.reserve(pkeys.size());

    for (const auto& pkey : pkeys) {
        const auto res = db.invoke_on(partitioner.shard_of(pkey.token()), [gs = global_schema_ptr(s), &pkey] (database& db) {
            return async([s = gs.get(), &pkey, &db] () mutable {
                auto accounter = db.get_result_memory_limiter().new_mutation_read(std::numeric_limits<size_t>::max()).get0();
                const auto cmd = query::read_command(s->id(), s->version(), s->full_slice(), query::max_rows);
                const auto range = dht::partition_range::make_singular(pkey);
                return make_foreign(std::make_unique<reconcilable_result>(
                    db.query_mutations(std::move(s), cmd, range, std::move(accounter), nullptr).get0()));
            });
        }).get0();

        BOOST_REQUIRE_EQUAL(res->partitions().size(), 1);
        results.emplace_back(res->partitions().front().mut().unfreeze(s));
    }

    return results;
}

using stateful_query = bool_class<class stateful>;

static std::pair<std::vector<mutation>, size_t>
read_all_partitions_with_paged_scan(distributed<database>& db, schema_ptr s, size_t page_size, stateful_query is_stateful,
        const std::function<void(size_t)>& page_hook) {
    const auto max_size = std::numeric_limits<uint64_t>::max();
    const auto query_uuid = is_stateful ? utils::make_random_uuid() : utils::UUID{};
    std::vector<mutation> results;
    auto cmd = query::read_command(s->id(), s->version(), s->full_slice(), page_size, gc_clock::now(), std::nullopt, query::max_partitions,
            query_uuid, true);

    // First page is special, needs to have `is_first_page` set.
    {
        auto res = query_mutations_on_all_shards(db, s, cmd, {query::full_partition_range}, nullptr, max_size).get0();
        for (auto& part : res->partitions()) {
            results.emplace_back(part.mut().unfreeze(s));
        }
        cmd.is_first_page = false;
    }

    size_t nrows = page_size;
    unsigned npages = 0;

    // Rest of the pages.
    // Loop until a page turns up with less rows than the limit.
    while (nrows == page_size) {
        page_hook(npages);
        if (is_stateful) {
            BOOST_REQUIRE(aggregate_querier_cache_stat(db, &query::querier_cache::stats::lookups) >= npages);
        }

        const auto& last_pkey = results.back().decorated_key();
        const auto& last_ckey = results.back().partition().clustered_rows().rbegin()->key();
        auto pkrange = dht::partition_range::make_starting_with(dht::partition_range::bound(last_pkey, true));
        auto ckrange = query::clustering_range::make_starting_with(query::clustering_range::bound(last_ckey, false));
        if (const auto& sr = cmd.slice.get_specific_ranges(); sr) {
            cmd.slice.clear_range(*s, sr->pk());
        }
        cmd.slice.set_range(*s, last_pkey.key(), {ckrange});

        auto res = query_mutations_on_all_shards(db, s, cmd, {pkrange}, nullptr, max_size).get0();

        if (res->partitions().empty()) {
            nrows = 0;
        } else {
            auto it = res->partitions().begin();
            auto end = res->partitions().end();
            auto first_mut = it->mut().unfreeze(s);

            nrows = first_mut.live_row_count();

            // The first partition of the new page may overlap with the last
            // partition of the last page.
            if (results.back().decorated_key().equal(*s, first_mut.decorated_key())) {
                results.back().apply(std::move(first_mut));
            } else {
                results.emplace_back(std::move(first_mut));
            }
            ++it;
            for (;it != end; ++it) {
                auto mut = it->mut().unfreeze(s);
                nrows += mut.live_row_count();
                results.emplace_back(std::move(mut));
            }
        }

        ++npages;
    }

    return std::pair(results, npages);
}

void check_results_are_equal(std::vector<mutation>& results1, std::vector<mutation>& results2) {
    BOOST_REQUIRE_EQUAL(results1.size(), results2.size());

    auto mut_less = [] (const mutation& a, const mutation& b) {
        return a.decorated_key().less_compare(*a.schema(), b.decorated_key());
    };
    boost::sort(results1, mut_less);
    boost::sort(results2, mut_less);
    for (unsigned i = 0; i < results1.size(); ++i) {
        BOOST_TEST_MESSAGE(format("Comparing mutation #{:d}", i));
        assert_that(results2[i]).is_equal_to(results1[i]);
    }
}

// Best run with SMP>=2
SEASTAR_THREAD_TEST_CASE(test_read_all) {
    do_with_cql_env([] (cql_test_env& env) -> future<> {
        using namespace std::chrono_literals;

        env.db().invoke_on_all([] (database& db) {
            db.set_querier_cache_entry_ttl(2s);
        }).get();

        auto [s, pkeys] = test::create_test_table(env, KEYSPACE_NAME, "test_read_all");

        // First read all partition-by-partition (not paged).
        auto results1 = read_all_partitions_one_by_one(env.db(), s, pkeys);

        // Then do a paged range-query, with reader caching
        auto results2 = read_all_partitions_with_paged_scan(env.db(), s, 4, stateful_query::yes, [&] (size_t) {
            check_cache_population(env.db(), 1);
            BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::drops), 0);
            BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::misses), 0);
        }).first;

        check_results_are_equal(results1, results2);

        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::drops), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::misses), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::time_based_evictions), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::resource_based_evictions), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::memory_based_evictions), 0);

        require_eventually_empty_caches(env.db());

        // Then do a paged range-query, without reader caching
        auto results3 = read_all_partitions_with_paged_scan(env.db(), s, 4, stateful_query::no, [&] (size_t) {
            check_cache_population(env.db(), 0);
        }).first;

        check_results_are_equal(results1, results3);

        return make_ready_future<>();
    }).get();
}

// Best run with SMP>=2
SEASTAR_THREAD_TEST_CASE(test_evict_a_shard_reader_on_each_page) {
    do_with_cql_env([] (cql_test_env& env) -> future<> {
        using namespace std::chrono_literals;

        env.db().invoke_on_all([] (database& db) {
            db.set_querier_cache_entry_ttl(2s);
        }).get();

        auto [s, pkeys] = test::create_test_table(env, KEYSPACE_NAME, "test_evict_a_shard_reader_on_each_page");

        // First read all partition-by-partition (not paged).
        auto results1 = read_all_partitions_one_by_one(env.db(), s, pkeys);

        // Then do a paged range-query
        auto [results2, npages] = read_all_partitions_with_paged_scan(env.db(), s, 4, stateful_query::yes, [&] (size_t page) {
            check_cache_population(env.db(), 1);

            env.db().invoke_on(page % smp::count, [&] (database& db) {
                db.get_querier_cache().evict_one();
            }).get();

            BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::drops), 0);
            BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::misses), page);
        });

        check_results_are_equal(results1, results2);

        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::drops), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::time_based_evictions), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::resource_based_evictions), npages);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::memory_based_evictions), 0);

        require_eventually_empty_caches(env.db());

        return make_ready_future<>();
    }).get();
}
<commit_msg>tests/multishard_mutation_query_test: refactor read_all_partitions_with_paged_scan()<commit_after>/*
 * Copyright (C) 2018 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "multishard_mutation_query.hh"
#include "schema_registry.hh"
#include "tests/cql_test_env.hh"
#include "tests/eventually.hh"
#include "tests/cql_assertions.hh"
#include "tests/mutation_assertions.hh"
#include "tests/test_table.hh"

#include <seastar/testing/thread_test_case.hh>

#include <experimental/source_location>

const sstring KEYSPACE_NAME = "multishard_mutation_query_test";

static uint64_t aggregate_querier_cache_stat(distributed<database>& db, uint64_t query::querier_cache::stats::*stat) {
    return map_reduce(boost::irange(0u, smp::count), [stat, &db] (unsigned shard) {
        return db.invoke_on(shard, [stat] (database& local_db) {
            auto& stats = local_db.get_querier_cache_stats();
            return stats.*stat;
        });
    }, 0, std::plus<size_t>()).get0();
}

static void check_cache_population(distributed<database>& db, size_t queriers,
        std::experimental::source_location sl = std::experimental::source_location::current()) {
    BOOST_TEST_MESSAGE(format("{}() called from {}() {}:{:d}", __FUNCTION__, sl.function_name(), sl.file_name(), sl.line()));

    parallel_for_each(boost::irange(0u, smp::count), [queriers, &db] (unsigned shard) {
        return db.invoke_on(shard, [queriers] (database& local_db) {
            auto& stats = local_db.get_querier_cache_stats();
            BOOST_REQUIRE_EQUAL(stats.population, queriers);
        });
    }).get0();
}

static void require_eventually_empty_caches(distributed<database>& db,
        std::experimental::source_location sl = std::experimental::source_location::current()) {
    BOOST_TEST_MESSAGE(format("{}() called from {}() {}:{:d}", __FUNCTION__, sl.function_name(), sl.file_name(), sl.line()));

    auto aggregated_population_is_zero = [&] () mutable {
        return aggregate_querier_cache_stat(db, &query::querier_cache::stats::population) == 0;
    };
    BOOST_REQUIRE(eventually_true(aggregated_population_is_zero));
}

// Best run with SMP>=2
SEASTAR_THREAD_TEST_CASE(test_abandoned_read) {
    do_with_cql_env([] (cql_test_env& env) -> future<> {
        using namespace std::chrono_literals;

        env.db().invoke_on_all([] (database& db) {
            db.set_querier_cache_entry_ttl(2s);
        }).get();

        auto [s, _] = test::create_test_table(env, KEYSPACE_NAME, "test_abandoned_read");
        (void)_;

        auto cmd = query::read_command(s->id(), s->version(), s->full_slice(), 7, gc_clock::now(), std::nullopt, query::max_partitions,
                utils::make_random_uuid(), true);

        query_mutations_on_all_shards(env.db(), s, cmd, {query::full_partition_range}, nullptr, std::numeric_limits<uint64_t>::max()).get();

        check_cache_population(env.db(), 1);

        sleep(2s).get();

        require_eventually_empty_caches(env.db());

        return make_ready_future<>();
    }).get();
}

static std::vector<mutation> read_all_partitions_one_by_one(distributed<database>& db, schema_ptr s, std::vector<dht::decorated_key> pkeys) {
    const auto& partitioner = dht::global_partitioner();
    std::vector<mutation> results;
    results.reserve(pkeys.size());

    for (const auto& pkey : pkeys) {
        const auto res = db.invoke_on(partitioner.shard_of(pkey.token()), [gs = global_schema_ptr(s), &pkey] (database& db) {
            return async([s = gs.get(), &pkey, &db] () mutable {
                auto accounter = db.get_result_memory_limiter().new_mutation_read(std::numeric_limits<size_t>::max()).get0();
                const auto cmd = query::read_command(s->id(), s->version(), s->full_slice(), query::max_rows);
                const auto range = dht::partition_range::make_singular(pkey);
                return make_foreign(std::make_unique<reconcilable_result>(
                    db.query_mutations(std::move(s), cmd, range, std::move(accounter), nullptr).get0()));
            });
        }).get0();

        BOOST_REQUIRE_EQUAL(res->partitions().size(), 1);
        results.emplace_back(res->partitions().front().mut().unfreeze(s));
    }

    return results;
}

using stateful_query = bool_class<class stateful>;

static std::pair<std::vector<mutation>, size_t>
read_partitions_with_paged_scan(distributed<database>& db, schema_ptr s, uint32_t page_size, uint64_t max_size, stateful_query is_stateful,
        const dht::partition_range& range, const query::partition_slice& slice, const std::function<void(size_t)>& page_hook = {}) {
    const auto query_uuid = is_stateful ? utils::make_random_uuid() : utils::UUID{};
    std::vector<mutation> results;
    auto cmd = query::read_command(s->id(), s->version(), slice, page_size, gc_clock::now(), std::nullopt, query::max_partitions, query_uuid, true);

    bool has_more = true;

    // First page is special, needs to have `is_first_page` set.
    {
        auto res = query_mutations_on_all_shards(db, s, cmd, {range}, nullptr, max_size).get0();
        for (auto& part : res->partitions()) {
            auto mut = part.mut().unfreeze(s);
            results.emplace_back(std::move(mut));
        }
        cmd.is_first_page = false;
        has_more = !res->partitions().empty();
    }

    if (!has_more) {
        return std::pair(results, 1);
    }

    unsigned npages = 0;

    const auto last_ckey_of = [] (const mutation& mut) -> std::optional<clustering_key> {
        if (mut.partition().clustered_rows().empty()) {
            return std::nullopt;
        }
        return mut.partition().clustered_rows().rbegin()->key();
    };

    auto last_pkey = results.back().decorated_key();
    auto last_ckey = last_ckey_of(results.back());

    // Rest of the pages. Loop until an empty page turns up. Not very
    // sophisticated but simple and safe.
    while (has_more) {
        if (page_hook) {
            page_hook(npages);
        }

        ++npages;

        auto pkrange = dht::partition_range(dht::partition_range::bound(last_pkey, last_ckey.has_value()), range.end());

        if (last_ckey) {
            auto ckranges = cmd.slice.default_row_ranges();
            query::trim_clustering_row_ranges_to(*s, ckranges, *last_ckey);
            cmd.slice.clear_range(*s, last_pkey.key());
            cmd.slice.clear_ranges();
            cmd.slice.set_range(*s, last_pkey.key(), std::move(ckranges));
        }

        auto res = query_mutations_on_all_shards(db, s, cmd, {pkrange}, nullptr, max_size).get0();

        if (is_stateful) {
            BOOST_REQUIRE(aggregate_querier_cache_stat(db, &query::querier_cache::stats::lookups) >= npages);
        }

        if (!res->partitions().empty()) {
            auto it = res->partitions().begin();
            auto end = res->partitions().end();
            auto first_mut = it->mut().unfreeze(s);

            last_pkey = first_mut.decorated_key();
            last_ckey = last_ckey_of(first_mut);

            // The first partition of the new page may overlap with the last
            // partition of the last page.
            if (results.back().decorated_key().equal(*s, first_mut.decorated_key())) {
                results.back().apply(std::move(first_mut));
            } else {
                results.emplace_back(std::move(first_mut));
            }
            ++it;
            for (;it != end; ++it) {
                auto mut = it->mut().unfreeze(s);
                last_pkey = mut.decorated_key();
                last_ckey = last_ckey_of(mut);
                results.emplace_back(std::move(mut));
            }
        }

        has_more = !res->partitions().empty();
    }

    return std::pair(results, npages);
}

static std::pair<std::vector<mutation>, size_t>
read_all_partitions_with_paged_scan(distributed<database>& db, schema_ptr s, uint32_t page_size, stateful_query is_stateful,
        const std::function<void(size_t)>& page_hook) {
    return read_partitions_with_paged_scan(db, s, page_size, std::numeric_limits<uint64_t>::max(), is_stateful, query::full_partition_range,
            s->full_slice(), page_hook);
}

void check_results_are_equal(std::vector<mutation>& results1, std::vector<mutation>& results2) {
    BOOST_REQUIRE_EQUAL(results1.size(), results2.size());

    auto mut_less = [] (const mutation& a, const mutation& b) {
        return a.decorated_key().less_compare(*a.schema(), b.decorated_key());
    };
    boost::sort(results1, mut_less);
    boost::sort(results2, mut_less);
    for (unsigned i = 0; i < results1.size(); ++i) {
        BOOST_TEST_MESSAGE(format("Comparing mutation #{:d}", i));
        assert_that(results2[i]).is_equal_to(results1[i]);
    }
}

// Best run with SMP>=2
SEASTAR_THREAD_TEST_CASE(test_read_all) {
    do_with_cql_env([] (cql_test_env& env) -> future<> {
        using namespace std::chrono_literals;

        env.db().invoke_on_all([] (database& db) {
            db.set_querier_cache_entry_ttl(2s);
        }).get();

        auto [s, pkeys] = test::create_test_table(env, KEYSPACE_NAME, "test_read_all");

        // First read all partition-by-partition (not paged).
        auto results1 = read_all_partitions_one_by_one(env.db(), s, pkeys);

        // Then do a paged range-query, with reader caching
        auto results2 = read_all_partitions_with_paged_scan(env.db(), s, 4, stateful_query::yes, [&] (size_t) {
            check_cache_population(env.db(), 1);
            BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::drops), 0);
            BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::misses), 0);
        }).first;

        check_results_are_equal(results1, results2);

        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::drops), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::misses), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::time_based_evictions), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::resource_based_evictions), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::memory_based_evictions), 0);

        require_eventually_empty_caches(env.db());

        // Then do a paged range-query, without reader caching
        auto results3 = read_all_partitions_with_paged_scan(env.db(), s, 4, stateful_query::no, [&] (size_t) {
            check_cache_population(env.db(), 0);
        }).first;

        check_results_are_equal(results1, results3);

        return make_ready_future<>();
    }).get();
}

// Best run with SMP>=2
SEASTAR_THREAD_TEST_CASE(test_evict_a_shard_reader_on_each_page) {
    do_with_cql_env([] (cql_test_env& env) -> future<> {
        using namespace std::chrono_literals;

        env.db().invoke_on_all([] (database& db) {
            db.set_querier_cache_entry_ttl(2s);
        }).get();

        auto [s, pkeys] = test::create_test_table(env, KEYSPACE_NAME, "test_evict_a_shard_reader_on_each_page");

        // First read all partition-by-partition (not paged).
        auto results1 = read_all_partitions_one_by_one(env.db(), s, pkeys);

        // Then do a paged range-query
        auto [results2, npages] = read_all_partitions_with_paged_scan(env.db(), s, 4, stateful_query::yes, [&] (size_t page) {
            check_cache_population(env.db(), 1);

            env.db().invoke_on(page % smp::count, [&] (database& db) {
                db.get_querier_cache().evict_one();
            }).get();

            BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::drops), 0);
            BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::misses), page);
        });

        check_results_are_equal(results1, results2);

        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::drops), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::time_based_evictions), 0);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::resource_based_evictions), npages);
        BOOST_REQUIRE_EQUAL(aggregate_querier_cache_stat(env.db(), &query::querier_cache::stats::memory_based_evictions), 0);

        require_eventually_empty_caches(env.db());

        return make_ready_future<>();
    }).get();
}
<|endoftext|>