Changeset 33a694 for ThirdParty/vmg/src/thirdparty/pugixml/pugixml.cpp

Timestamp:

May 5, 2017, 2:17:23 PM (8 years ago)

Author:

Frederik Heber <frederik.heber@…>

Branches:

Action_Thermostats, Add_AtomRandomPerturbation, Add_RotateAroundBondAction, Add_SelectAtomByNameAction, Adding_Graph_to_ChangeBondActions, Adding_MD_integration_tests, Adding_StructOpt_integration_tests, AutomationFragmentation_failures, Candidate_v1.6.1, ChangeBugEmailaddress, ChangingTestPorts, ChemicalSpaceEvaluator, Docu_Python_wait, EmpiricalPotential_contain_HomologyGraph_documentation, Enhance_userguide, Enhanced_StructuralOptimization, Enhanced_StructuralOptimization_continued, Example_ManyWaysToTranslateAtom, Exclude_Hydrogens_annealWithBondGraph, Fix_ChronosMutex, Fix_StatusMsg, Fix_StepWorldTime_single_argument, Fix_Verbose_Codepatterns, ForceAnnealing_goodresults, ForceAnnealing_oldresults, ForceAnnealing_tocheck, ForceAnnealing_with_BondGraph, ForceAnnealing_with_BondGraph_continued, ForceAnnealing_with_BondGraph_continued_betteresults, ForceAnnealing_with_BondGraph_contraction-expansion, GeometryObjects, Gui_displays_atomic_force_velocity, IndependentFragmentGrids_IntegrationTest, JobMarket_RobustOnKillsSegFaults, JobMarket_StableWorkerPool, PythonUI_with_named_parameters, QtGui_reactivate_TimeChanged_changes, Recreated_GuiChecks, RotateToPrincipalAxisSystem_UndoRedo, StoppableMakroAction, TremoloParser_IncreasedPrecision, TremoloParser_MultipleTimesteps, Ubuntu_1604_changes

Children:

dde50d

Parents:

6798a5

git-author:

Frederik Heber <heber@…> (04/05/17 17:17:45)

git-committer:

Frederik Heber <frederik.heber@…> (05/05/17 14:17:23)

Message:

Updated pugixml in ThirdParty/vmg and src from 1.0 to 1.8.

• this solved compilation issues with 1.0 seemingly from too recent gcc and c++11/14/.. changes.

File:

• ThirdParty/vmg/src/thirdparty/pugixml/pugixml.cpp (modified) (377 diffs)

ThirdParty/vmg/src/thirdparty/pugixml/pugixml.cpp

@@ -1 +1 @@
 /**
- * pugixml parser - version 1.0
+ * pugixml parser - version 1.8
  * --------------------------------------------------------
- * Copyright (C) 2006-2010, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
+ * Copyright (C) 2006-2016, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
  * Report bugs and download new versions at http://pugixml.org/
  *
@@ -12 +12 @@
  */
 
+#ifndef SOURCE_PUGIXML_CPP
+#define SOURCE_PUGIXML_CPP
+
 #include "pugixml.hpp"
 
@@ -18 +21 @@
 #include <string.h>
 #include <assert.h>
-#include <setjmp.h>
-#include <wchar.h>
+#include <limits.h>
+
+#ifdef PUGIXML_WCHAR_MODE
+#       include <wchar.h>
+#endif
 
 #ifndef PUGIXML_NO_XPATH
 #       include <math.h>
 #       include <float.h>
+#       ifdef PUGIXML_NO_EXCEPTIONS
+#               include <setjmp.h>
+#       endif
 #endif
 
@@ -36 +45 @@
 
 #ifdef _MSC_VER
+#       pragma warning(push)
 #       pragma warning(disable: 4127) // conditional expression is constant
 #       pragma warning(disable: 4324) // structure was padded due to __declspec(align())
@@ -41 +51 @@
 #       pragma warning(disable: 4702) // unreachable code
 #       pragma warning(disable: 4996) // this function or variable may be unsafe
+#       pragma warning(disable: 4793) // function compiled as native: presence of '_setjmp' makes a function unmanaged
 #endif
 
 #ifdef __INTEL_COMPILER
-#       pragma warning(disable: 177) // function was declared but never referenced 
+#       pragma warning(disable: 177) // function was declared but never referenced
 #       pragma warning(disable: 279) // controlling expression is constant
 #       pragma warning(disable: 1478 1786) // function was declared "deprecated"
+#       pragma warning(disable: 1684) // conversion from pointer to same-sized integral type
 #endif
 
+#if defined(__BORLANDC__) && defined(PUGIXML_HEADER_ONLY)
+#       pragma warn -8080 // symbol is declared but never used; disabling this inside push/pop bracket does not make the warning go away
+#endif
+
 #ifdef __BORLANDC__
+#       pragma option push
 #       pragma warn -8008 // condition is always false
 #       pragma warn -8066 // unreachable code
@@ -55 +72 @@
 
 #ifdef __SNC__
+// Using diag_push/diag_pop does not disable the warnings inside templates due to a compiler bug
 #       pragma diag_suppress=178 // function was declared but never referenced
 #       pragma diag_suppress=237 // controlling expression is constant
 #endif
 
-// uintptr_t
-#if !defined(_MSC_VER) || _MSC_VER >= 1600
-#       include <stdint.h>
-#else
-#       if _MSC_VER < 1300
-// No native uintptr_t in MSVC6
-typedef size_t uintptr_t;
-#       endif
-typedef unsigned __int8 uint8_t;
-typedef unsigned __int16 uint16_t;
-typedef unsigned __int32 uint32_t;
-typedef __int32 int32_t;
-#endif
-
 // Inlining controls
 #if defined(_MSC_VER) && _MSC_VER >= 1300
-#       define PUGIXML_NO_INLINE __declspec(noinline)
+#       define PUGI__NO_INLINE __declspec(noinline)
 #elif defined(__GNUC__)
-#       define PUGIXML_NO_INLINE __attribute__((noinline))
+#       define PUGI__NO_INLINE __attribute__((noinline))
 #else
-#       define PUGIXML_NO_INLINE 
+#       define PUGI__NO_INLINE
 #endif
 
+// Branch weight controls
+#if defined(__GNUC__)
+#       define PUGI__UNLIKELY(cond) __builtin_expect(cond, 0)
+#else
+#       define PUGI__UNLIKELY(cond) (cond)
+#endif
+
 // Simple static assertion
-#define STATIC_ASSERT(cond) { static const char condition_failed[(cond) ? 1 : -1] = {0}; (void)condition_failed[0]; }
+#define PUGI__STATIC_ASSERT(cond) { static const char condition_failed[(cond) ? 1 : -1] = {0}; (void)condition_failed[0]; }
 
 // Digital Mars C++ bug workaround for passing char loaded from memory via stack
 #ifdef __DMC__
-#       define DMC_VOLATILE volatile
+#       define PUGI__DMC_VOLATILE volatile
 #else
-#       define DMC_VOLATILE
+#       define PUGI__DMC_VOLATILE
 #endif
 
-using namespace pugi;
+// Borland C++ bug workaround for not defining ::memcpy depending on header include order (can't always use std::memcpy because some compilers don't have it at all)
+#if defined(__BORLANDC__) && !defined(__MEM_H_USING_LIST)
+using std::memcpy;
+using std::memmove;
+using std::memset;
+#endif
+
+// Some MinGW versions have headers that erroneously omit LLONG_MIN/LLONG_MAX/ULLONG_MAX definitions in strict ANSI mode
+#if defined(PUGIXML_HAS_LONG_LONG) && defined(__MINGW32__) && defined(__STRICT_ANSI__) && !defined(LLONG_MAX) && !defined(LLONG_MIN) && !defined(ULLONG_MAX)
+#       define LLONG_MAX 9223372036854775807LL
+#       define LLONG_MIN (-LLONG_MAX-1)
+#       define ULLONG_MAX (2ULL*LLONG_MAX+1)
+#endif
+
+// In some environments MSVC is a compiler but the CRT lacks certain MSVC-specific features
+#if defined(_MSC_VER) && !defined(__S3E__)
+#       define PUGI__MSVC_CRT_VERSION _MSC_VER
+#endif
+
+#ifdef PUGIXML_HEADER_ONLY
+#       define PUGI__NS_BEGIN namespace pugi { namespace impl {
+#       define PUGI__NS_END } }
+#       define PUGI__FN inline
+#       define PUGI__FN_NO_INLINE inline
+#else
+#       if defined(_MSC_VER) && _MSC_VER < 1300 // MSVC6 seems to have an amusing bug with anonymous namespaces inside namespaces
+#               define PUGI__NS_BEGIN namespace pugi { namespace impl {
+#               define PUGI__NS_END } }
+#       else
+#               define PUGI__NS_BEGIN namespace pugi { namespace impl { namespace {
+#               define PUGI__NS_END } } }
+#       endif
+#       define PUGI__FN
+#       define PUGI__FN_NO_INLINE PUGI__NO_INLINE
+#endif
+
+// uintptr_t
+#if (defined(_MSC_VER) && _MSC_VER < 1600) || (defined(__BORLANDC__) && __BORLANDC__ < 0x561)
+namespace pugi
+{
+#       ifndef _UINTPTR_T_DEFINED
+        typedef size_t uintptr_t;
+#       endif
+
+        typedef unsigned __int8 uint8_t;
+        typedef unsigned __int16 uint16_t;
+        typedef unsigned __int32 uint32_t;
+}
+#else
+#       include <stdint.h>
+#endif
 
 // Memory allocation
-namespace
-{
-        void* default_allocate(size_t size)
+PUGI__NS_BEGIN
+        PUGI__FN void* default_allocate(size_t size)
         {
                 return malloc(size);
         }
 
-        void default_deallocate(void* ptr)
+        PUGI__FN void default_deallocate(void* ptr)
         {
                 free(ptr);
         }
 
-        allocation_function global_allocate = default_allocate;
-        deallocation_function global_deallocate = default_deallocate;
-}
+        template <typename T>
+        struct xml_memory_management_function_storage
+        {
+                static allocation_function allocate;
+                static deallocation_function deallocate;
+        };
+
+        // Global allocation functions are stored in class statics so that in header mode linker deduplicates them
+        // Without a template<> we'll get multiple definitions of the same static
+        template <typename T> allocation_function xml_memory_management_function_storage<T>::allocate = default_allocate;
+        template <typename T> deallocation_function xml_memory_management_function_storage<T>::deallocate = default_deallocate;
+
+        typedef xml_memory_management_function_storage<int> xml_memory;
+PUGI__NS_END
 
 // String utilities
-namespace
-{
+PUGI__NS_BEGIN
         // Get string length
-        size_t strlength(const char_t* s)
+        PUGI__FN size_t strlength(const char_t* s)
         {
                 assert(s);
@@ -127 +197 @@
 
         // Compare two strings
-        bool strequal(const char_t* src, const char_t* dst)
+        PUGI__FN bool strequal(const char_t* src, const char_t* dst)
         {
                 assert(src && dst);
@@ -139 +209 @@
 
         // Compare lhs with [rhs_begin, rhs_end)
-        bool strequalrange(const char_t* lhs, const char_t* rhs, size_t count)
+        PUGI__FN bool strequalrange(const char_t* lhs, const char_t* rhs, size_t count)
         {
                 for (size_t i = 0; i < count; ++i)
                         if (lhs[i] != rhs[i])
                                 return false;
-        
+
                 return lhs[count] == 0;
         }
-        
-#ifdef PUGIXML_WCHAR_MODE
-        // Convert string to wide string, assuming all symbols are ASCII
-        void widen_ascii(wchar_t* dest, const char* source)
-        {
-                for (const char* i = source; *i; ++i) *dest++ = *i;
-                *dest = 0;
-        }
-#endif
-}
-
-#if !defined(PUGIXML_NO_STL) || !defined(PUGIXML_NO_XPATH)
-// auto_ptr-like buffer holder for exception recovery
-namespace
-{
-        struct buffer_holder
-        {
-                void* data;
-                void (*deleter)(void*);
-
-                buffer_holder(void* data, void (*deleter)(void*)): data(data), deleter(deleter)
-                {
-                }
-
-                ~buffer_holder()
+
+        // Get length of wide string, even if CRT lacks wide character support
+        PUGI__FN size_t strlength_wide(const wchar_t* s)
+        {
+                assert(s);
+
+        #ifdef PUGIXML_WCHAR_MODE
+                return wcslen(s);
+        #else
+                const wchar_t* end = s;
+                while (*end) end++;
+                return static_cast<size_t>(end - s);
+        #endif
+        }
+PUGI__NS_END
+
+// auto_ptr-like object for exception recovery
+PUGI__NS_BEGIN
+        template <typename T> struct auto_deleter
+        {
+                typedef void (*D)(T*);
+
+                T* data;
+                D deleter;
+
+                auto_deleter(T* data_, D deleter_): data(data_), deleter(deleter_)
+                {
+                }
+
+                ~auto_deleter()
                 {
                         if (data) deleter(data);
                 }
 
-                void* release()
-                {
-                        void* result = data;
+                T* release()
+                {
+                        T* result = data;
                         data = 0;
                         return result;
                 }
         };
-}
+PUGI__NS_END
+
+#ifdef PUGIXML_COMPACT
+PUGI__NS_BEGIN
+        class compact_hash_table
+        {
+        public:
+                compact_hash_table(): _items(0), _capacity(0), _count(0)
+                {
+                }
+
+                void clear()
+                {
+                        if (_items)
+                        {
+                                xml_memory::deallocate(_items);
+                                _items = 0;
+                                _capacity = 0;
+                                _count = 0;
+                        }
+                }
+
+                void** find(const void* key)
+                {
+                        assert(key);
+
+                        if (_capacity == 0) return 0;
+
+                        size_t hashmod = _capacity - 1;
+                        size_t bucket = hash(key) & hashmod;
+
+                        for (size_t probe = 0; probe <= hashmod; ++probe)
+                        {
+                                item_t& probe_item = _items[bucket];
+
+                                if (probe_item.key == key)
+                                        return &probe_item.value;
+
+                                if (probe_item.key == 0)
+                                        return 0;
+
+                                // hash collision, quadratic probing
+                                bucket = (bucket + probe + 1) & hashmod;
+                        }
+
+                        assert(false && "Hash table is full");
+                        return 0;
+                }
+
+                void** insert(const void* key)
+                {
+                        assert(key);
+                        assert(_capacity != 0 && _count < _capacity - _capacity / 4);
+
+                        size_t hashmod = _capacity - 1;
+                        size_t bucket = hash(key) & hashmod;
+
+                        for (size_t probe = 0; probe <= hashmod; ++probe)
+                        {
+                                item_t& probe_item = _items[bucket];
+
+                                if (probe_item.key == 0)
+                                {
+                                        probe_item.key = key;
+                                        _count++;
+                                        return &probe_item.value;
+                                }
+
+                                if (probe_item.key == key)
+                                        return &probe_item.value;
+
+                                // hash collision, quadratic probing
+                                bucket = (bucket + probe + 1) & hashmod;
+                        }
+
+                        assert(false && "Hash table is full");
+                        return 0;
+                }
+
+                bool reserve()
+                {
+                        if (_count + 16 >= _capacity - _capacity / 4)
+                                return rehash();
+
+                        return true;
+                }
+
+        private:
+                struct item_t
+                {
+                        const void* key;
+                        void* value;
+                };
+
+                item_t* _items;
+                size_t _capacity;
+
+                size_t _count;
+
+                bool rehash();
+
+                static unsigned int hash(const void* key)
+                {
+                        unsigned int h = static_cast<unsigned int>(reinterpret_cast<uintptr_t>(key));
+
+                        // MurmurHash3 32-bit finalizer
+                        h ^= h >> 16;
+                        h *= 0x85ebca6bu;
+                        h ^= h >> 13;
+                        h *= 0xc2b2ae35u;
+                        h ^= h >> 16;
+
+                        return h;
+                }
+        };
+
+        PUGI__FN_NO_INLINE bool compact_hash_table::rehash()
+        {
+                compact_hash_table rt;
+                rt._capacity = (_capacity == 0) ? 32 : _capacity * 2;
+                rt._items = static_cast<item_t*>(xml_memory::allocate(sizeof(item_t) * rt._capacity));
+
+                if (!rt._items)
+                        return false;
+
+                memset(rt._items, 0, sizeof(item_t) * rt._capacity);
+
+                for (size_t i = 0; i < _capacity; ++i)
+                        if (_items[i].key)
+                                *rt.insert(_items[i].key) = _items[i].value;
+
+                if (_items)
+                        xml_memory::deallocate(_items);
+
+                _capacity = rt._capacity;
+                _items = rt._items;
+
+                assert(_count == rt._count);
+
+                return true;
+        }
+
+PUGI__NS_END
 #endif
 
-namespace
-{
-        static const size_t xml_memory_page_size = 32768;
-
-        static const uintptr_t xml_memory_page_alignment = 32;
-        static const uintptr_t xml_memory_page_pointer_mask = ~(xml_memory_page_alignment - 1);
-        static const uintptr_t xml_memory_page_name_allocated_mask = 16;
-        static const uintptr_t xml_memory_page_value_allocated_mask = 8;
-        static const uintptr_t xml_memory_page_type_mask = 7;
+PUGI__NS_BEGIN
+#ifdef PUGIXML_COMPACT
+        static const uintptr_t xml_memory_block_alignment = 4;
+#else
+        static const uintptr_t xml_memory_block_alignment = sizeof(void*);
+#endif
+
+        // extra metadata bits
+        static const uintptr_t xml_memory_page_contents_shared_mask = 64;
+        static const uintptr_t xml_memory_page_name_allocated_mask = 32;
+        static const uintptr_t xml_memory_page_value_allocated_mask = 16;
+        static const uintptr_t xml_memory_page_type_mask = 15;
+
+        // combined masks for string uniqueness
+        static const uintptr_t xml_memory_page_name_allocated_or_shared_mask = xml_memory_page_name_allocated_mask | xml_memory_page_contents_shared_mask;
+        static const uintptr_t xml_memory_page_value_allocated_or_shared_mask = xml_memory_page_value_allocated_mask | xml_memory_page_contents_shared_mask;
+
+#ifdef PUGIXML_COMPACT
+        #define PUGI__GETHEADER_IMPL(object, page, flags) // unused
+        #define PUGI__GETPAGE_IMPL(header) (header).get_page()
+#else
+        #define PUGI__GETHEADER_IMPL(object, page, flags) (((reinterpret_cast<char*>(object) - reinterpret_cast<char*>(page)) << 8) | (flags))
+        // this macro casts pointers through void* to avoid 'cast increases required alignment of target type' warnings
+        #define PUGI__GETPAGE_IMPL(header) static_cast<impl::xml_memory_page*>(const_cast<void*>(static_cast<const void*>(reinterpret_cast<const char*>(&header) - (header >> 8))))
+#endif
+
+        #define PUGI__GETPAGE(n) PUGI__GETPAGE_IMPL((n)->header)
+        #define PUGI__NODETYPE(n) static_cast<xml_node_type>((n)->header & impl::xml_memory_page_type_mask)
 
         struct xml_allocator;
@@ -202 +438 @@
                 static xml_memory_page* construct(void* memory)
                 {
-                        if (!memory) return 0; //$ redundant, left for performance
-
                         xml_memory_page* result = static_cast<xml_memory_page*>(memory);
 
                         result->allocator = 0;
-                        result->memory = 0;
                         result->prev = 0;
                         result->next = 0;
@@ -213 +446 @@
                         result->freed_size = 0;
 
+                #ifdef PUGIXML_COMPACT
+                        result->compact_string_base = 0;
+                        result->compact_shared_parent = 0;
+                        result->compact_page_marker = 0;
+                #endif
+
                         return result;
                 }
 
                 xml_allocator* allocator;
-
-                void* memory;
 
                 xml_memory_page* prev;
@@ -226 +463 @@
                 size_t freed_size;
 
-                char data[1];
+        #ifdef PUGIXML_COMPACT
+                char_t* compact_string_base;
+                void* compact_shared_parent;
+                uint32_t* compact_page_marker;
+        #endif
         };
+
+        static const size_t xml_memory_page_size =
+        #ifdef PUGIXML_MEMORY_PAGE_SIZE
+                (PUGIXML_MEMORY_PAGE_SIZE)
+        #else
+                32768
+        #endif
+                - sizeof(xml_memory_page);
 
         struct xml_memory_string_header
@@ -239 +488 @@
                 xml_allocator(xml_memory_page* root): _root(root), _busy_size(root->busy_size)
                 {
+                #ifdef PUGIXML_COMPACT
+                        _hash = 0;
+                #endif
                 }
 
                 xml_memory_page* allocate_page(size_t data_size)
                 {
-                        size_t size = offsetof(xml_memory_page, data) + data_size;
+                        size_t size = sizeof(xml_memory_page) + data_size;
 
                         // allocate block with some alignment, leaving memory for worst-case padding
-                        void* memory = global_allocate(size + xml_memory_page_alignment);
+                        void* memory = xml_memory::allocate(size);
                         if (!memory) return 0;
 
-                        // align upwards to page boundary
-                        void* page_memory = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(memory) + (xml_memory_page_alignment - 1)) & ~(xml_memory_page_alignment - 1));
-
                         // prepare page structure
-                        xml_memory_page* page = xml_memory_page::construct(page_memory);
-
-                        page->memory = memory;
+                        xml_memory_page* page = xml_memory_page::construct(memory);
+                        assert(page);
+
                         page->allocator = _root->allocator;
 
@@ -263 +512 @@
                 static void deallocate_page(xml_memory_page* page)
                 {
-                        global_deallocate(page->memory);
+                        xml_memory::deallocate(page);
                 }
 
@@ -270 +519 @@
                 void* allocate_memory(size_t size, xml_memory_page*& out_page)
                 {
-                        if (_busy_size + size > xml_memory_page_size) return allocate_memory_oob(size, out_page);
-
-                        void* buf = _root->data + _busy_size;
+                        if (PUGI__UNLIKELY(_busy_size + size > xml_memory_page_size))
+                                return allocate_memory_oob(size, out_page);
+
+                        void* buf = reinterpret_cast<char*>(_root) + sizeof(xml_memory_page) + _busy_size;
 
                         _busy_size += size;
@@ -281 +531 @@
                 }
 
+        #ifdef PUGIXML_COMPACT
+                void* allocate_object(size_t size, xml_memory_page*& out_page)
+                {
+                        void* result = allocate_memory(size + sizeof(uint32_t), out_page);
+                        if (!result) return 0;
+
+                        // adjust for marker
+                        ptrdiff_t offset = static_cast<char*>(result) - reinterpret_cast<char*>(out_page->compact_page_marker);
+
+                        if (PUGI__UNLIKELY(static_cast<uintptr_t>(offset) >= 256 * xml_memory_block_alignment))
+                        {
+                                // insert new marker
+                                uint32_t* marker = static_cast<uint32_t*>(result);
+
+                                *marker = static_cast<uint32_t>(reinterpret_cast<char*>(marker) - reinterpret_cast<char*>(out_page));
+                                out_page->compact_page_marker = marker;
+
+                                // since we don't reuse the page space until we reallocate it, we can just pretend that we freed the marker block
+                                // this will make sure deallocate_memory correctly tracks the size
+                                out_page->freed_size += sizeof(uint32_t);
+
+                                return marker + 1;
+                        }
+                        else
+                        {
+                                // roll back uint32_t part
+                                _busy_size -= sizeof(uint32_t);
+
+                                return result;
+                        }
+                }
+        #else
+                void* allocate_object(size_t size, xml_memory_page*& out_page)
+                {
+                        return allocate_memory(size, out_page);
+                }
+        #endif
+
                 void deallocate_memory(void* ptr, size_t size, xml_memory_page* page)
                 {
                         if (page == _root) page->busy_size = _busy_size;
 
-                        assert(ptr >= page->data && ptr < page->data + page->busy_size);
+                        assert(ptr >= reinterpret_cast<char*>(page) + sizeof(xml_memory_page) && ptr < reinterpret_cast<char*>(page) + sizeof(xml_memory_page) + page->busy_size);
                         (void)!ptr;
 
@@ -298 +586 @@
 
                                         // top page freed, just reset sizes
-                                        page->busy_size = page->freed_size = 0;
+                                        page->busy_size = 0;
+                                        page->freed_size = 0;
+
+                                #ifdef PUGIXML_COMPACT
+                                        // reset compact state to maximize efficiency
+                                        page->compact_string_base = 0;
+                                        page->compact_shared_parent = 0;
+                                        page->compact_page_marker = 0;
+                                #endif
+
                                         _busy_size = 0;
                                 }
@@ -318 +615 @@
                 char_t* allocate_string(size_t length)
                 {
+                        static const size_t max_encoded_offset = (1 << 16) * xml_memory_block_alignment;
+
+                        PUGI__STATIC_ASSERT(xml_memory_page_size <= max_encoded_offset);
+
                         // allocate memory for string and header block
                         size_t size = sizeof(xml_memory_string_header) + length * sizeof(char_t);
-                        
-                        // round size up to pointer alignment boundary
-                        size_t full_size = (size + (sizeof(void*) - 1)) & ~(sizeof(void*) - 1);
+
+                        // round size up to block alignment boundary
+                        size_t full_size = (size + (xml_memory_block_alignment - 1)) & ~(xml_memory_block_alignment - 1);
 
                         xml_memory_page* page;
@@ -330 +631 @@
 
                         // setup header
-                        ptrdiff_t page_offset = reinterpret_cast<char*>(header) - page->data;
-
-                        assert(page_offset >= 0 && page_offset < (1 << 16));
-                        header->page_offset = static_cast<uint16_t>(page_offset);
+                        ptrdiff_t page_offset = reinterpret_cast<char*>(header) - reinterpret_cast<char*>(page) - sizeof(xml_memory_page);
+
+                        assert(page_offset % xml_memory_block_alignment == 0);
+                        assert(page_offset >= 0 && static_cast<size_t>(page_offset) < max_encoded_offset);
+                        header->page_offset = static_cast<uint16_t>(static_cast<size_t>(page_offset) / xml_memory_block_alignment);
 
                         // full_size == 0 for large strings that occupy the whole page
-                        assert(full_size < (1 << 16) || (page->busy_size == full_size && page_offset == 0));
-                        header->full_size = static_cast<uint16_t>(full_size < (1 << 16) ? full_size : 0);
-
-                        return reinterpret_cast<char_t*>(header + 1);
+                        assert(full_size % xml_memory_block_alignment == 0);
+                        assert(full_size < max_encoded_offset || (page->busy_size == full_size && page_offset == 0));
+                        header->full_size = static_cast<uint16_t>(full_size < max_encoded_offset ? full_size / xml_memory_block_alignment : 0);
+
+                        // round-trip through void* to avoid 'cast increases required alignment of target type' warning
+                        // header is guaranteed a pointer-sized alignment, which should be enough for char_t
+                        return static_cast<char_t*>(static_cast<void*>(header + 1));
                 }
 
                 void deallocate_string(char_t* string)
                 {
+                        // this function casts pointers through void* to avoid 'cast increases required alignment of target type' warnings
+                        // we're guaranteed the proper (pointer-sized) alignment on the input string if it was allocated via allocate_string
+
                         // get header
-                        xml_memory_string_header* header = reinterpret_cast<xml_memory_string_header*>(string) - 1;
+                        xml_memory_string_header* header = static_cast<xml_memory_string_header*>(static_cast<void*>(string)) - 1;
+                        assert(header);
 
                         // deallocate
-                        size_t page_offset = offsetof(xml_memory_page, data) + header->page_offset;
-                        xml_memory_page* page = reinterpret_cast<xml_memory_page*>(reinterpret_cast<char*>(header) - page_offset);
+                        size_t page_offset = sizeof(xml_memory_page) + header->page_offset * xml_memory_block_alignment;
+                        xml_memory_page* page = reinterpret_cast<xml_memory_page*>(static_cast<void*>(reinterpret_cast<char*>(header) - page_offset));
 
                         // if full_size == 0 then this string occupies the whole page
-                        size_t full_size = header->full_size == 0 ? page->busy_size : header->full_size;
+                        size_t full_size = header->full_size == 0 ? page->busy_size : header->full_size * xml_memory_block_alignment;
 
                         deallocate_memory(header, full_size, page);
+                }
+
+                bool reserve()
+                {
+                #ifdef PUGIXML_COMPACT
+                        return _hash->reserve();
+                #else
+                        return true;
+                #endif
                 }
 
                 xml_memory_page* _root;
                 size_t _busy_size;
+
+        #ifdef PUGIXML_COMPACT
+                compact_hash_table* _hash;
+        #endif
         };
 
-        PUGIXML_NO_INLINE void* xml_allocator::allocate_memory_oob(size_t size, xml_memory_page*& out_page)
+        PUGI__FN_NO_INLINE void* xml_allocator::allocate_memory_oob(size_t size, xml_memory_page*& out_page)
         {
                 const size_t large_allocation_threshold = xml_memory_page_size / 4;
 
                 xml_memory_page* page = allocate_page(size <= large_allocation_threshold ? xml_memory_page_size : size);
+                out_page = page;
+
                 if (!page) return 0;
 
@@ -390 +714 @@
                         _root->prev->next = page;
                         _root->prev = page;
-                }
-
-                // allocate inside page
-                page->busy_size = size;
-
-                out_page = page;
-                return page->data;
-        }
-}
-
+
+                        page->busy_size = size;
+                }
+
+                return reinterpret_cast<char*>(page) + sizeof(xml_memory_page);
+        }
+PUGI__NS_END
+
+#ifdef PUGIXML_COMPACT
+PUGI__NS_BEGIN
+        static const uintptr_t compact_alignment_log2 = 2;
+        static const uintptr_t compact_alignment = 1 << compact_alignment_log2;
+
+        class compact_header
+        {
+        public:
+                compact_header(xml_memory_page* page, unsigned int flags)
+                {
+                        PUGI__STATIC_ASSERT(xml_memory_block_alignment == compact_alignment);
+
+                        ptrdiff_t offset = (reinterpret_cast<char*>(this) - reinterpret_cast<char*>(page->compact_page_marker));
+                        assert(offset % compact_alignment == 0 && static_cast<uintptr_t>(offset) < 256 * compact_alignment);
+
+                        _page = static_cast<unsigned char>(offset >> compact_alignment_log2);
+                        _flags = static_cast<unsigned char>(flags);
+                }
+
+                void operator&=(uintptr_t mod)
+                {
+                        _flags &= static_cast<unsigned char>(mod);
+                }
+
+                void operator|=(uintptr_t mod)
+                {
+                        _flags |= static_cast<unsigned char>(mod);
+                }
+
+                uintptr_t operator&(uintptr_t mod) const
+                {
+                        return _flags & mod;
+                }
+
+                xml_memory_page* get_page() const
+                {
+                        // round-trip through void* to silence 'cast increases required alignment of target type' warnings
+                        const char* page_marker = reinterpret_cast<const char*>(this) - (_page << compact_alignment_log2);
+                        const char* page = page_marker - *reinterpret_cast<const uint32_t*>(static_cast<const void*>(page_marker));
+
+                        return const_cast<xml_memory_page*>(reinterpret_cast<const xml_memory_page*>(static_cast<const void*>(page)));
+                }
+
+        private:
+                unsigned char _page;
+                unsigned char _flags;
+        };
+
+        PUGI__FN xml_memory_page* compact_get_page(const void* object, int header_offset)
+        {
+                const compact_header* header = reinterpret_cast<const compact_header*>(static_cast<const char*>(object) - header_offset);
+
+                return header->get_page();
+        }
+
+        template <int header_offset, typename T> PUGI__FN_NO_INLINE T* compact_get_value(const void* object)
+        {
+                return static_cast<T*>(*compact_get_page(object, header_offset)->allocator->_hash->find(object));
+        }
+
+        template <int header_offset, typename T> PUGI__FN_NO_INLINE void compact_set_value(const void* object, T* value)
+        {
+                *compact_get_page(object, header_offset)->allocator->_hash->insert(object) = value;
+        }
+
+        template <typename T, int header_offset, int start = -126> class compact_pointer
+        {
+        public:
+                compact_pointer(): _data(0)
+                {
+                }
+
+                void operator=(const compact_pointer& rhs)
+                {
+                        *this = rhs + 0;
+                }
+
+                void operator=(T* value)
+                {
+                        if (value)
+                        {
+                                // value is guaranteed to be compact-aligned; 'this' is not
+                                // our decoding is based on 'this' aligned to compact alignment downwards (see operator T*)
+                                // so for negative offsets (e.g. -3) we need to adjust the diff by compact_alignment - 1 to
+                                // compensate for arithmetic shift rounding for negative values
+                                ptrdiff_t diff = reinterpret_cast<char*>(value) - reinterpret_cast<char*>(this);
+                                ptrdiff_t offset = ((diff + int(compact_alignment - 1)) >> compact_alignment_log2) - start;
+
+                                if (static_cast<uintptr_t>(offset) <= 253)
+                                        _data = static_cast<unsigned char>(offset + 1);
+                                else
+                                {
+                                        compact_set_value<header_offset>(this, value);
+
+                                        _data = 255;
+                                }
+                        }
+                        else
+                                _data = 0;
+                }
+
+                operator T*() const
+                {
+                        if (_data)
+                        {
+                                if (_data < 255)
+                                {
+                                        uintptr_t base = reinterpret_cast<uintptr_t>(this) & ~(compact_alignment - 1);
+
+                                        return reinterpret_cast<T*>(base + ((_data - 1 + start) << compact_alignment_log2));
+                                }
+                                else
+                                        return compact_get_value<header_offset, T>(this);
+                        }
+                        else
+                                return 0;
+                }
+
+                T* operator->() const
+                {
+                        return *this;
+                }
+
+        private:
+                unsigned char _data;
+        };
+
+        template <typename T, int header_offset> class compact_pointer_parent
+        {
+        public:
+                compact_pointer_parent(): _data(0)
+                {
+                }
+
+                void operator=(const compact_pointer_parent& rhs)
+                {
+                        *this = rhs + 0;
+                }
+
+                void operator=(T* value)
+                {
+                        if (value)
+                        {
+                                // value is guaranteed to be compact-aligned; 'this' is not
+                                // our decoding is based on 'this' aligned to compact alignment downwards (see operator T*)
+                                // so for negative offsets (e.g. -3) we need to adjust the diff by compact_alignment - 1 to
+                                // compensate for arithmetic shift behavior for negative values
+                                ptrdiff_t diff = reinterpret_cast<char*>(value) - reinterpret_cast<char*>(this);
+                                ptrdiff_t offset = ((diff + int(compact_alignment - 1)) >> compact_alignment_log2) + 65533;
+
+                                if (static_cast<uintptr_t>(offset) <= 65533)
+                                {
+                                        _data = static_cast<unsigned short>(offset + 1);
+                                }
+                                else
+                                {
+                                        xml_memory_page* page = compact_get_page(this, header_offset);
+
+                                        if (PUGI__UNLIKELY(page->compact_shared_parent == 0))
+                                                page->compact_shared_parent = value;
+
+                                        if (page->compact_shared_parent == value)
+                                        {
+                                                _data = 65534;
+                                        }
+                                        else
+                                        {
+                                                compact_set_value<header_offset>(this, value);
+
+                                                _data = 65535;
+                                        }
+                                }
+                        }
+                        else
+                        {
+                                _data = 0;
+                        }
+                }
+
+                operator T*() const
+                {
+                        if (_data)
+                        {
+                                if (_data < 65534)
+                                {
+                                        uintptr_t base = reinterpret_cast<uintptr_t>(this) & ~(compact_alignment - 1);
+
+                                        return reinterpret_cast<T*>(base + ((_data - 1 - 65533) << compact_alignment_log2));
+                                }
+                                else if (_data == 65534)
+                                        return static_cast<T*>(compact_get_page(this, header_offset)->compact_shared_parent);
+                                else
+                                        return compact_get_value<header_offset, T>(this);
+                        }
+                        else
+                                return 0;
+                }
+
+                T* operator->() const
+                {
+                        return *this;
+                }
+
+        private:
+                uint16_t _data;
+        };
+
+        template <int header_offset, int base_offset> class compact_string
+        {
+        public:
+                compact_string(): _data(0)
+                {
+                }
+
+                void operator=(const compact_string& rhs)
+                {
+                        *this = rhs + 0;
+                }
+
+                void operator=(char_t* value)
+                {
+                        if (value)
+                        {
+                                xml_memory_page* page = compact_get_page(this, header_offset);
+
+                                if (PUGI__UNLIKELY(page->compact_string_base == 0))
+                                        page->compact_string_base = value;
+
+                                ptrdiff_t offset = value - page->compact_string_base;
+
+                                if (static_cast<uintptr_t>(offset) < (65535 << 7))
+                                {
+                                        // round-trip through void* to silence 'cast increases required alignment of target type' warnings
+                                        uint16_t* base = reinterpret_cast<uint16_t*>(static_cast<void*>(reinterpret_cast<char*>(this) - base_offset));
+
+                                        if (*base == 0)
+                                        {
+                                                *base = static_cast<uint16_t>((offset >> 7) + 1);
+                                                _data = static_cast<unsigned char>((offset & 127) + 1);
+                                        }
+                                        else
+                                        {
+                                                ptrdiff_t remainder = offset - ((*base - 1) << 7);
+
+                                                if (static_cast<uintptr_t>(remainder) <= 253)
+                                                {
+                                                        _data = static_cast<unsigned char>(remainder + 1);
+                                                }
+                                                else
+                                                {
+                                                        compact_set_value<header_offset>(this, value);
+
+                                                        _data = 255;
+                                                }
+                                        }
+                                }
+                                else
+                                {
+                                        compact_set_value<header_offset>(this, value);
+
+                                        _data = 255;
+                                }
+                        }
+                        else
+                        {
+                                _data = 0;
+                        }
+                }
+
+                operator char_t*() const
+                {
+                        if (_data)
+                        {
+                                if (_data < 255)
+                                {
+                                        xml_memory_page* page = compact_get_page(this, header_offset);
+
+                                        // round-trip through void* to silence 'cast increases required alignment of target type' warnings
+                                        const uint16_t* base = reinterpret_cast<const uint16_t*>(static_cast<const void*>(reinterpret_cast<const char*>(this) - base_offset));
+                                        assert(*base);
+
+                                        ptrdiff_t offset = ((*base - 1) << 7) + (_data - 1);
+
+                                        return page->compact_string_base + offset;
+                                }
+                                else
+                                {
+                                        return compact_get_value<header_offset, char_t>(this);
+                                }
+                        }
+                        else
+                                return 0;
+                }
+
+        private:
+                unsigned char _data;
+        };
+PUGI__NS_END
+#endif
+
+#ifdef PUGIXML_COMPACT
 namespace pugi
 {
-        /// A 'name=value' XML attribute structure.
         struct xml_attribute_struct
         {
-                /// Default ctor
-                xml_attribute_struct(xml_memory_page* page): header(reinterpret_cast<uintptr_t>(page)), name(0), value(0), prev_attribute_c(0), next_attribute(0)
-                {
-                }
-
-                uintptr_t header;
-
-                char_t* name;   ///< Pointer to attribute name.
-                char_t* value;  ///< Pointer to attribute value.
-
-                xml_attribute_struct* prev_attribute_c; ///< Previous attribute (cyclic list)
-                xml_attribute_struct* next_attribute;   ///< Next attribute
+                xml_attribute_struct(impl::xml_memory_page* page): header(page, 0), namevalue_base(0)
+                {
+                        PUGI__STATIC_ASSERT(sizeof(xml_attribute_struct) == 8);
+                }
+
+                impl::compact_header header;
+
+                uint16_t namevalue_base;
+
+                impl::compact_string<4, 2> name;
+                impl::compact_string<5, 3> value;
+
+                impl::compact_pointer<xml_attribute_struct, 6> prev_attribute_c;
+                impl::compact_pointer<xml_attribute_struct, 7, 0> next_attribute;
         };
 
-        /// An XML document tree node.
         struct xml_node_struct
         {
-                /// Default ctor
-                /// \param type - node type
-                xml_node_struct(xml_memory_page* page, xml_node_type type): header(reinterpret_cast<uintptr_t>(page) | (type - 1)), parent(0), name(0), value(0), first_child(0), prev_sibling_c(0), next_sibling(0), first_attribute(0)
-                {
-                }
-
-                uintptr_t header;
-
-                xml_node_struct*                parent;                                 ///< Pointer to parent
-
-                char_t*                                 name;                                   ///< Pointer to element name.
-                char_t*                                 value;                                  ///< Pointer to any associated string data.
-
-                xml_node_struct*                first_child;                    ///< First child
-                
-                xml_node_struct*                prev_sibling_c;                 ///< Left brother (cyclic list)
-                xml_node_struct*                next_sibling;                   ///< Right brother
-                
-                xml_attribute_struct*   first_attribute;                ///< First attribute
+                xml_node_struct(impl::xml_memory_page* page, xml_node_type type): header(page, type), namevalue_base(0)
+                {
+                        PUGI__STATIC_ASSERT(sizeof(xml_node_struct) == 12);
+                }
+
+                impl::compact_header header;
+
+                uint16_t namevalue_base;
+
+                impl::compact_string<4, 2> name;
+                impl::compact_string<5, 3> value;
+
+                impl::compact_pointer_parent<xml_node_struct, 6> parent;
+
+                impl::compact_pointer<xml_node_struct, 8, 0> first_child;
+
+                impl::compact_pointer<xml_node_struct,  9>    prev_sibling_c;
+                impl::compact_pointer<xml_node_struct, 10, 0> next_sibling;
+
+                impl::compact_pointer<xml_attribute_struct, 11, 0> first_attribute;
         };
 }
-
-namespace
+#else
+namespace pugi
 {
+        struct xml_attribute_struct
+        {
+                xml_attribute_struct(impl::xml_memory_page* page): name(0), value(0), prev_attribute_c(0), next_attribute(0)
+                {
+                        header = PUGI__GETHEADER_IMPL(this, page, 0);
+                }
+
+                uintptr_t header;
+
+                char_t* name;
+                char_t* value;
+
+                xml_attribute_struct* prev_attribute_c;
+                xml_attribute_struct* next_attribute;
+        };
+
+        struct xml_node_struct
+        {
+                xml_node_struct(impl::xml_memory_page* page, xml_node_type type): name(0), value(0), parent(0), first_child(0), prev_sibling_c(0), next_sibling(0), first_attribute(0)
+                {
+                        header = PUGI__GETHEADER_IMPL(this, page, type);
+                }
+
+                uintptr_t header;
+
+                char_t* name;
+                char_t* value;
+
+                xml_node_struct* parent;
+
+                xml_node_struct* first_child;
+
+                xml_node_struct* prev_sibling_c;
+                xml_node_struct* next_sibling;
+
+                xml_attribute_struct* first_attribute;
+        };
+}
+#endif
+
+PUGI__NS_BEGIN
+        struct xml_extra_buffer
+        {
+                char_t* buffer;
+                xml_extra_buffer* next;
+        };
+
         struct xml_document_struct: public xml_node_struct, public xml_allocator
         {
-                xml_document_struct(xml_memory_page* page): xml_node_struct(page, node_document), xml_allocator(page), buffer(0)
+                xml_document_struct(xml_memory_page* page): xml_node_struct(page, node_document), xml_allocator(page), buffer(0), extra_buffers(0)
                 {
                 }
 
                 const char_t* buffer;
+
+                xml_extra_buffer* extra_buffers;
+
+        #ifdef PUGIXML_COMPACT
+                compact_hash_table hash;
+        #endif
         };
 
-        static inline xml_allocator& get_allocator(const xml_node_struct* node)
-        {
-                assert(node);
-
-                return *reinterpret_cast<xml_memory_page*>(node->header & xml_memory_page_pointer_mask)->allocator;
-        }
-}
+        template <typename Object> inline xml_allocator& get_allocator(const Object* object)
+        {
+                assert(object);
+
+                return *PUGI__GETPAGE(object)->allocator;
+        }
+
+        template <typename Object> inline xml_document_struct& get_document(const Object* object)
+        {
+                assert(object);
+
+                return *static_cast<xml_document_struct*>(PUGI__GETPAGE(object)->allocator);
+        }
+PUGI__NS_END
 
 // Low-level DOM operations
-namespace
-{
+PUGI__NS_BEGIN
         inline xml_attribute_struct* allocate_attribute(xml_allocator& alloc)
         {
                 xml_memory_page* page;
-                void* memory = alloc.allocate_memory(sizeof(xml_attribute_struct), page);
+                void* memory = alloc.allocate_object(sizeof(xml_attribute_struct), page);
+                if (!memory) return 0;
 
                 return new (memory) xml_attribute_struct(page);
@@ -477 +1161 @@
         {
                 xml_memory_page* page;
-                void* memory = alloc.allocate_memory(sizeof(xml_node_struct), page);
+                void* memory = alloc.allocate_object(sizeof(xml_node_struct), page);
+                if (!memory) return 0;
 
                 return new (memory) xml_node_struct(page, type);
@@ -484 +1169 @@
         inline void destroy_attribute(xml_attribute_struct* a, xml_allocator& alloc)
         {
-                uintptr_t header = a->header;
-
-                if (header & xml_memory_page_name_allocated_mask) alloc.deallocate_string(a->name);
-                if (header & xml_memory_page_value_allocated_mask) alloc.deallocate_string(a->value);
-
-                alloc.deallocate_memory(a, sizeof(xml_attribute_struct), reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask));
+                if (a->header & impl::xml_memory_page_name_allocated_mask)
+                        alloc.deallocate_string(a->name);
+
+                if (a->header & impl::xml_memory_page_value_allocated_mask)
+                        alloc.deallocate_string(a->value);
+
+                alloc.deallocate_memory(a, sizeof(xml_attribute_struct), PUGI__GETPAGE(a));
         }
 
         inline void destroy_node(xml_node_struct* n, xml_allocator& alloc)
         {
-                uintptr_t header = n->header;
-
-                if (header & xml_memory_page_name_allocated_mask) alloc.deallocate_string(n->name);
-                if (header & xml_memory_page_value_allocated_mask) alloc.deallocate_string(n->value);
+                if (n->header & impl::xml_memory_page_name_allocated_mask)
+                        alloc.deallocate_string(n->name);
+
+                if (n->header & impl::xml_memory_page_value_allocated_mask)
+                        alloc.deallocate_string(n->value);
 
                 for (xml_attribute_struct* attr = n->first_attribute; attr; )
@@ -517 +1204 @@
                 }
 
-                alloc.deallocate_memory(n, sizeof(xml_node_struct), reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask));
-        }
-
-        PUGIXML_NO_INLINE xml_node_struct* append_node(xml_node_struct* node, xml_allocator& alloc, xml_node_type type = node_element)
-        {
+                alloc.deallocate_memory(n, sizeof(xml_node_struct), PUGI__GETPAGE(n));
+        }
+
+        inline void append_node(xml_node_struct* child, xml_node_struct* node)
+        {
+                child->parent = node;
+
+                xml_node_struct* head = node->first_child;
+
+                if (head)
+                {
+                        xml_node_struct* tail = head->prev_sibling_c;
+
+                        tail->next_sibling = child;
+                        child->prev_sibling_c = tail;
+                        head->prev_sibling_c = child;
+                }
+                else
+                {
+                        node->first_child = child;
+                        child->prev_sibling_c = child;
+                }
+        }
+
+        inline void prepend_node(xml_node_struct* child, xml_node_struct* node)
+        {
+                child->parent = node;
+
+                xml_node_struct* head = node->first_child;
+
+                if (head)
+                {
+                        child->prev_sibling_c = head->prev_sibling_c;
+                        head->prev_sibling_c = child;
+                }
+                else
+                        child->prev_sibling_c = child;
+
+                child->next_sibling = head;
+                node->first_child = child;
+        }
+
+        inline void insert_node_after(xml_node_struct* child, xml_node_struct* node)
+        {
+                xml_node_struct* parent = node->parent;
+
+                child->parent = parent;
+
+                if (node->next_sibling)
+                        node->next_sibling->prev_sibling_c = child;
+                else
+                        parent->first_child->prev_sibling_c = child;
+
+                child->next_sibling = node->next_sibling;
+                child->prev_sibling_c = node;
+
+                node->next_sibling = child;
+        }
+
+        inline void insert_node_before(xml_node_struct* child, xml_node_struct* node)
+        {
+                xml_node_struct* parent = node->parent;
+
+                child->parent = parent;
+
+                if (node->prev_sibling_c->next_sibling)
+                        node->prev_sibling_c->next_sibling = child;
+                else
+                        parent->first_child = child;
+
+                child->prev_sibling_c = node->prev_sibling_c;
+                child->next_sibling = node;
+
+                node->prev_sibling_c = child;
+        }
+
+        inline void remove_node(xml_node_struct* node)
+        {
+                xml_node_struct* parent = node->parent;
+
+                if (node->next_sibling)
+                        node->next_sibling->prev_sibling_c = node->prev_sibling_c;
+                else
+                        parent->first_child->prev_sibling_c = node->prev_sibling_c;
+
+                if (node->prev_sibling_c->next_sibling)
+                        node->prev_sibling_c->next_sibling = node->next_sibling;
+                else
+                        parent->first_child = node->next_sibling;
+
+                node->parent = 0;
+                node->prev_sibling_c = 0;
+                node->next_sibling = 0;
+        }
+
+        inline void append_attribute(xml_attribute_struct* attr, xml_node_struct* node)
+        {
+                xml_attribute_struct* head = node->first_attribute;
+
+                if (head)
+                {
+                        xml_attribute_struct* tail = head->prev_attribute_c;
+
+                        tail->next_attribute = attr;
+                        attr->prev_attribute_c = tail;
+                        head->prev_attribute_c = attr;
+                }
+                else
+                {
+                        node->first_attribute = attr;
+                        attr->prev_attribute_c = attr;
+                }
+        }
+
+        inline void prepend_attribute(xml_attribute_struct* attr, xml_node_struct* node)
+        {
+                xml_attribute_struct* head = node->first_attribute;
+
+                if (head)
+                {
+                        attr->prev_attribute_c = head->prev_attribute_c;
+                        head->prev_attribute_c = attr;
+                }
+                else
+                        attr->prev_attribute_c = attr;
+
+                attr->next_attribute = head;
+                node->first_attribute = attr;
+        }
+
+        inline void insert_attribute_after(xml_attribute_struct* attr, xml_attribute_struct* place, xml_node_struct* node)
+        {
+                if (place->next_attribute)
+                        place->next_attribute->prev_attribute_c = attr;
+                else
+                        node->first_attribute->prev_attribute_c = attr;
+
+                attr->next_attribute = place->next_attribute;
+                attr->prev_attribute_c = place;
+                place->next_attribute = attr;
+        }
+
+        inline void insert_attribute_before(xml_attribute_struct* attr, xml_attribute_struct* place, xml_node_struct* node)
+        {
+                if (place->prev_attribute_c->next_attribute)
+                        place->prev_attribute_c->next_attribute = attr;
+                else
+                        node->first_attribute = attr;
+
+                attr->prev_attribute_c = place->prev_attribute_c;
+                attr->next_attribute = place;
+                place->prev_attribute_c = attr;
+        }
+
+        inline void remove_attribute(xml_attribute_struct* attr, xml_node_struct* node)
+        {
+                if (attr->next_attribute)
+                        attr->next_attribute->prev_attribute_c = attr->prev_attribute_c;
+                else
+                        node->first_attribute->prev_attribute_c = attr->prev_attribute_c;
+
+                if (attr->prev_attribute_c->next_attribute)
+                        attr->prev_attribute_c->next_attribute = attr->next_attribute;
+                else
+                        node->first_attribute = attr->next_attribute;
+
+                attr->prev_attribute_c = 0;
+                attr->next_attribute = 0;
+        }
+
+        PUGI__FN_NO_INLINE xml_node_struct* append_new_node(xml_node_struct* node, xml_allocator& alloc, xml_node_type type = node_element)
+        {
+                if (!alloc.reserve()) return 0;
+
                 xml_node_struct* child = allocate_node(alloc, type);
                 if (!child) return 0;
 
-                child->parent = node;
-
-                xml_node_struct* first_child = node->first_child;
-                        
-                if (first_child)
-                {
-                        xml_node_struct* last_child = first_child->prev_sibling_c;
-
-                        last_child->next_sibling = child;
-                        child->prev_sibling_c = last_child;
-                        first_child->prev_sibling_c = child;
-                }
-                else
-                {
-                        node->first_child = child;
-                        child->prev_sibling_c = child;
-                }
-                        
+                append_node(child, node);
+
                 return child;
         }
 
-        PUGIXML_NO_INLINE xml_attribute_struct* append_attribute_ll(xml_node_struct* node, xml_allocator& alloc)
-        {
-                xml_attribute_struct* a = allocate_attribute(alloc);
-                if (!a) return 0;
-
-                xml_attribute_struct* first_attribute = node->first_attribute;
-
-                if (first_attribute)
-                {
-                        xml_attribute_struct* last_attribute = first_attribute->prev_attribute_c;
-
-                        last_attribute->next_attribute = a;
-                        a->prev_attribute_c = last_attribute;
-                        first_attribute->prev_attribute_c = a;
-                }
-                else
-                {
-                        node->first_attribute = a;
-                        a->prev_attribute_c = a;
-                }
-                        
-                return a;
-        }
-}
+        PUGI__FN_NO_INLINE xml_attribute_struct* append_new_attribute(xml_node_struct* node, xml_allocator& alloc)
+        {
+                if (!alloc.reserve()) return 0;
+
+                xml_attribute_struct* attr = allocate_attribute(alloc);
+                if (!attr) return 0;
+
+                append_attribute(attr, node);
+
+                return attr;
+        }
+PUGI__NS_END
 
 // Helper classes for code generation
-namespace
-{
+PUGI__NS_BEGIN
         struct opt_false
         {
@@ -583 +1410 @@
                 enum { value = 1 };
         };
-}
+PUGI__NS_END
 
 // Unicode utilities
-namespace
-{
+PUGI__NS_BEGIN
         inline uint16_t endian_swap(uint16_t value)
         {
@@ -692 +1518 @@
                 static value_type high(value_type result, uint32_t ch)
                 {
-                        uint32_t msh = (uint32_t)(ch - 0x10000) >> 10;
-                        uint32_t lsh = (uint32_t)(ch - 0x10000) & 0x3ff;
+                        uint32_t msh = static_cast<uint32_t>(ch - 0x10000) >> 10;
+                        uint32_t lsh = static_cast<uint32_t>(ch - 0x10000) & 0x3ff;
 
                         result[0] = static_cast<uint16_t>(0xD800 + msh);
@@ -748 +1574 @@
         };
 
-        template <size_t size> struct wchar_selector;
-
-        template <> struct wchar_selector<2>
-        {
-                typedef uint16_t type;
-                typedef utf16_counter counter;
-                typedef utf16_writer writer;
+        struct latin1_writer
+        {
+                typedef uint8_t* value_type;
+
+                static value_type low(value_type result, uint32_t ch)
+                {
+                        *result = static_cast<uint8_t>(ch > 255 ? '?' : ch);
+
+                        return result + 1;
+                }
+
+                static value_type high(value_type result, uint32_t ch)
+                {
+                        (void)ch;
+
+                        *result = '?';
+
+                        return result + 1;
+                }
         };
 
-        template <> struct wchar_selector<4>
-        {
-                typedef uint32_t type;
-                typedef utf32_counter counter;
-                typedef utf32_writer writer;
-        };
-
-        typedef wchar_selector<sizeof(wchar_t)>::counter wchar_counter;
-        typedef wchar_selector<sizeof(wchar_t)>::writer wchar_writer;
-
-        template <typename Traits, typename opt_swap = opt_false> struct utf_decoder
-        {
-                static inline typename Traits::value_type decode_utf8_block(const uint8_t* data, size_t size, typename Traits::value_type result)
+        struct utf8_decoder
+        {
+                typedef uint8_t type;
+
+                template <typename Traits> static inline typename Traits::value_type process(const uint8_t* data, size_t size, typename Traits::value_type result, Traits)
                 {
                         const uint8_t utf8_byte_mask = 0x3f;
@@ -787 +1617 @@
                                         if ((reinterpret_cast<uintptr_t>(data) & 3) == 0)
                                         {
-                                                while (size >= 4 && (*reinterpret_cast<const uint32_t*>(data) & 0x80808080) == 0)
+                                                // round-trip through void* to silence 'cast increases required alignment of target type' warnings
+                                                while (size >= 4 && (*static_cast<const uint32_t*>(static_cast<const void*>(data)) & 0x80808080) == 0)
                                                 {
                                                         result = Traits::low(result, data[0]);
@@ -799 +1630 @@
                                 }
                                 // 110xxxxx -> U+0080..U+07FF
-                                else if ((unsigned)(lead - 0xC0) < 0x20 && size >= 2 && (data[1] & 0xc0) == 0x80)
+                                else if (static_cast<unsigned int>(lead - 0xC0) < 0x20 && size >= 2 && (data[1] & 0xc0) == 0x80)
                                 {
                                         result = Traits::low(result, ((lead & ~0xC0) << 6) | (data[1] & utf8_byte_mask));
@@ -806 +1637 @@
                                 }
                                 // 1110xxxx -> U+0800-U+FFFF
-                                else if ((unsigned)(lead - 0xE0) < 0x10 && size >= 3 && (data[1] & 0xc0) == 0x80 && (data[2] & 0xc0) == 0x80)
+                                else if (static_cast<unsigned int>(lead - 0xE0) < 0x10 && size >= 3 && (data[1] & 0xc0) == 0x80 && (data[2] & 0xc0) == 0x80)
                                 {
                                         result = Traits::low(result, ((lead & ~0xE0) << 12) | ((data[1] & utf8_byte_mask) << 6) | (data[2] & utf8_byte_mask));
@@ -813 +1644 @@
                                 }
                                 // 11110xxx -> U+10000..U+10FFFF
-                                else if ((unsigned)(lead - 0xF0) < 0x08 && size >= 4 && (data[1] & 0xc0) == 0x80 && (data[2] & 0xc0) == 0x80 && (data[3] & 0xc0) == 0x80)
+                                else if (static_cast<unsigned int>(lead - 0xF0) < 0x08 && size >= 4 && (data[1] & 0xc0) == 0x80 && (data[2] & 0xc0) == 0x80 && (data[3] & 0xc0) == 0x80)
                                 {
                                         result = Traits::high(result, ((lead & ~0xF0) << 18) | ((data[1] & utf8_byte_mask) << 12) | ((data[2] & utf8_byte_mask) << 6) | (data[3] & utf8_byte_mask));
@@ -829 +1660 @@
                         return result;
                 }
-
-                static inline typename Traits::value_type decode_utf16_block(const uint16_t* data, size_t size, typename Traits::value_type result)
-                {
-                        const uint16_t* end = data + size;
-
-                        while (data < end)
+        };
+
+        template <typename opt_swap> struct utf16_decoder
+        {
+                typedef uint16_t type;
+
+                template <typename Traits> static inline typename Traits::value_type process(const uint16_t* data, size_t size, typename Traits::value_type result, Traits)
+                {
+                        while (size)
                         {
                                 uint16_t lead = opt_swap::value ? endian_swap(*data) : *data;
@@ -843 +1677 @@
                                         result = Traits::low(result, lead);
                                         data += 1;
+                                        size -= 1;
                                 }
                                 // U+E000..U+FFFF
-                                else if ((unsigned)(lead - 0xE000) < 0x2000)
+                                else if (static_cast<unsigned int>(lead - 0xE000) < 0x2000)
                                 {
                                         result = Traits::low(result, lead);
                                         data += 1;
+                                        size -= 1;
                                 }
                                 // surrogate pair lead
-                                else if ((unsigned)(lead - 0xD800) < 0x400 && data + 1 < end)
+                                else if (static_cast<unsigned int>(lead - 0xD800) < 0x400 && size >= 2)
                                 {
                                         uint16_t next = opt_swap::value ? endian_swap(data[1]) : data[1];
 
-                                        if ((unsigned)(next - 0xDC00) < 0x400)
+                                        if (static_cast<unsigned int>(next - 0xDC00) < 0x400)
                                         {
                                                 result = Traits::high(result, 0x10000 + ((lead & 0x3ff) << 10) + (next & 0x3ff));
                                                 data += 2;
+                                                size -= 2;
                                         }
                                         else
                                         {
                                                 data += 1;
+                                                size -= 1;
                                         }
                                 }
@@ -868 +1706 @@
                                 {
                                         data += 1;
+                                        size -= 1;
                                 }
                         }
@@ -873 +1712 @@
                         return result;
                 }
-
-                static inline typename Traits::value_type decode_utf32_block(const uint32_t* data, size_t size, typename Traits::value_type result)
-                {
-                        const uint32_t* end = data + size;
-
-                        while (data < end)
+        };
+
+        template <typename opt_swap> struct utf32_decoder
+        {
+                typedef uint32_t type;
+
+                template <typename Traits> static inline typename Traits::value_type process(const uint32_t* data, size_t size, typename Traits::value_type result, Traits)
+                {
+                        while (size)
                         {
                                 uint32_t lead = opt_swap::value ? endian_swap(*data) : *data;
@@ -887 +1729 @@
                                         result = Traits::low(result, lead);
                                         data += 1;
+                                        size -= 1;
                                 }
                                 // U+10000..U+10FFFF
@@ -893 +1736 @@
                                         result = Traits::high(result, lead);
                                         data += 1;
+                                        size -= 1;
                                 }
                         }
@@ -900 +1744 @@
         };
 
-        template <typename T> inline void convert_utf_endian_swap(T* result, const T* data, size_t length)
-        {
-                for (size_t i = 0; i < length; ++i) result[i] = endian_swap(data[i]);
-        }
-
-        inline void convert_wchar_endian_swap(wchar_t* result, const wchar_t* data, size_t length)
-        {
-                for (size_t i = 0; i < length; ++i) result[i] = static_cast<wchar_t>(endian_swap(static_cast<wchar_selector<sizeof(wchar_t)>::type>(data[i])));
-        }
-}
-
-namespace
-{       
+        struct latin1_decoder
+        {
+                typedef uint8_t type;
+
+                template <typename Traits> static inline typename Traits::value_type process(const uint8_t* data, size_t size, typename Traits::value_type result, Traits)
+                {
+                        while (size)
+                        {
+                                result = Traits::low(result, *data);
+                                data += 1;
+                                size -= 1;
+                        }
+
+                        return result;
+                }
+        };
+
+        template <size_t size> struct wchar_selector;
+
+        template <> struct wchar_selector<2>
+        {
+                typedef uint16_t type;
+                typedef utf16_counter counter;
+                typedef utf16_writer writer;
+                typedef utf16_decoder<opt_false> decoder;
+        };
+
+        template <> struct wchar_selector<4>
+        {
+                typedef uint32_t type;
+                typedef utf32_counter counter;
+                typedef utf32_writer writer;
+                typedef utf32_decoder<opt_false> decoder;
+        };
+
+        typedef wchar_selector<sizeof(wchar_t)>::counter wchar_counter;
+        typedef wchar_selector<sizeof(wchar_t)>::writer wchar_writer;
+
+        struct wchar_decoder
+        {
+                typedef wchar_t type;
+
+                template <typename Traits> static inline typename Traits::value_type process(const wchar_t* data, size_t size, typename Traits::value_type result, Traits traits)
+                {
+                        typedef wchar_selector<sizeof(wchar_t)>::decoder decoder;
+
+                        return decoder::process(reinterpret_cast<const typename decoder::type*>(data), size, result, traits);
+                }
+        };
+
+#ifdef PUGIXML_WCHAR_MODE
+        PUGI__FN void convert_wchar_endian_swap(wchar_t* result, const wchar_t* data, size_t length)
+        {
+                for (size_t i = 0; i < length; ++i)
+                        result[i] = static_cast<wchar_t>(endian_swap(static_cast<wchar_selector<sizeof(wchar_t)>::type>(data[i])));
+        }
+#endif
+PUGI__NS_END
+
+PUGI__NS_BEGIN
         enum chartype_t
         {
@@ -925 +1816 @@
         };
 
-        const unsigned char chartype_table[256] =
+        static const unsigned char chartype_table[256] =
         {
                 55,  0,   0,   0,   0,   0,   0,   0,      0,   12,  12,  0,   0,   63,  0,   0,   // 0-15
@@ -954 +1845 @@
                 ctx_symbol = 16                   // Any symbol > 127, a-z, A-Z, 0-9, _, -, .
         };
-        
-        const unsigned char chartypex_table[256] =
+
+        static const unsigned char chartypex_table[256] =
         {
                 3,  3,  3,  3,  3,  3,  3,  3,     3,  0,  2,  3,  3,  2,  3,  3,     // 0-15
@@ -976 +1867 @@
                 20, 20, 20, 20, 20, 20, 20, 20,    20, 20, 20, 20, 20, 20, 20, 20
         };
-        
+
 #ifdef PUGIXML_WCHAR_MODE
-        #define IS_CHARTYPE_IMPL(c, ct, table) ((static_cast<unsigned int>(c) < 128 ? table[static_cast<unsigned int>(c)] : table[128]) & (ct))
+        #define PUGI__IS_CHARTYPE_IMPL(c, ct, table) ((static_cast<unsigned int>(c) < 128 ? table[static_cast<unsigned int>(c)] : table[128]) & (ct))
 #else
-        #define IS_CHARTYPE_IMPL(c, ct, table) (table[static_cast<unsigned char>(c)] & (ct))
+        #define PUGI__IS_CHARTYPE_IMPL(c, ct, table) (table[static_cast<unsigned char>(c)] & (ct))
 #endif
 
-        #define IS_CHARTYPE(c, ct) IS_CHARTYPE_IMPL(c, ct, chartype_table)
-        #define IS_CHARTYPEX(c, ct) IS_CHARTYPE_IMPL(c, ct, chartypex_table)
-
-        bool is_little_endian()
+        #define PUGI__IS_CHARTYPE(c, ct) PUGI__IS_CHARTYPE_IMPL(c, ct, chartype_table)
+        #define PUGI__IS_CHARTYPEX(c, ct) PUGI__IS_CHARTYPE_IMPL(c, ct, chartypex_table)
+
+        PUGI__FN bool is_little_endian()
         {
                 unsigned int ui = 1;
@@ -993 +1884 @@
         }
 
-        xml_encoding get_wchar_encoding()
-        {
-                STATIC_ASSERT(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4);
+        PUGI__FN xml_encoding get_wchar_encoding()
+        {
+                PUGI__STATIC_ASSERT(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4);
 
                 if (sizeof(wchar_t) == 2)
                         return is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
-                else 
+                else
                         return is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
         }
 
-        xml_encoding guess_buffer_encoding(uint8_t d0, uint8_t d1, uint8_t d2, uint8_t d3)
-        {
+        PUGI__FN bool parse_declaration_encoding(const uint8_t* data, size_t size, const uint8_t*& out_encoding, size_t& out_length)
+        {
+        #define PUGI__SCANCHAR(ch) { if (offset >= size || data[offset] != ch) return false; offset++; }
+        #define PUGI__SCANCHARTYPE(ct) { while (offset < size && PUGI__IS_CHARTYPE(data[offset], ct)) offset++; }
+
+                // check if we have a non-empty XML declaration
+                if (size < 6 || !((data[0] == '<') & (data[1] == '?') & (data[2] == 'x') & (data[3] == 'm') & (data[4] == 'l') && PUGI__IS_CHARTYPE(data[5], ct_space)))
+                        return false;
+
+                // scan XML declaration until the encoding field
+                for (size_t i = 6; i + 1 < size; ++i)
+                {
+                        // declaration can not contain ? in quoted values
+                        if (data[i] == '?')
+                                return false;
+
+                        if (data[i] == 'e' && data[i + 1] == 'n')
+                        {
+                                size_t offset = i;
+
+                                // encoding follows the version field which can't contain 'en' so this has to be the encoding if XML is well formed
+                                PUGI__SCANCHAR('e'); PUGI__SCANCHAR('n'); PUGI__SCANCHAR('c'); PUGI__SCANCHAR('o');
+                                PUGI__SCANCHAR('d'); PUGI__SCANCHAR('i'); PUGI__SCANCHAR('n'); PUGI__SCANCHAR('g');
+
+                                // S? = S?
+                                PUGI__SCANCHARTYPE(ct_space);
+                                PUGI__SCANCHAR('=');
+                                PUGI__SCANCHARTYPE(ct_space);
+
+                                // the only two valid delimiters are ' and "
+                                uint8_t delimiter = (offset < size && data[offset] == '"') ? '"' : '\'';
+
+                                PUGI__SCANCHAR(delimiter);
+
+                                size_t start = offset;
+
+                                out_encoding = data + offset;
+
+                                PUGI__SCANCHARTYPE(ct_symbol);
+
+                                out_length = offset - start;
+
+                                PUGI__SCANCHAR(delimiter);
+
+                                return true;
+                        }
+                }
+
+                return false;
+
+        #undef PUGI__SCANCHAR
+        #undef PUGI__SCANCHARTYPE
+        }
+
+        PUGI__FN xml_encoding guess_buffer_encoding(const uint8_t* data, size_t size)
+        {
+                // skip encoding autodetection if input buffer is too small
+                if (size < 4) return encoding_utf8;
+
+                uint8_t d0 = data[0], d1 = data[1], d2 = data[2], d3 = data[3];
+
                 // look for BOM in first few bytes
                 if (d0 == 0 && d1 == 0 && d2 == 0xfe && d3 == 0xff) return encoding_utf32_be;
@@ -1017 +1967 @@
                 if (d0 == 0 && d1 == 0x3c && d2 == 0 && d3 == 0x3f) return encoding_utf16_be;
                 if (d0 == 0x3c && d1 == 0 && d2 == 0x3f && d3 == 0) return encoding_utf16_le;
-                if (d0 == 0x3c && d1 == 0x3f && d2 == 0x78 && d3 == 0x6d) return encoding_utf8;
 
                 // look for utf16 < followed by node name (this may fail, but is better than utf8 since it's zero terminated so early)
@@ -1023 +1972 @@
                 if (d0 == 0x3c && d1 == 0) return encoding_utf16_le;
 
-                // no known BOM detected, assume utf8
+                // no known BOM detected; parse declaration
+                const uint8_t* enc = 0;
+                size_t enc_length = 0;
+
+                if (d0 == 0x3c && d1 == 0x3f && d2 == 0x78 && d3 == 0x6d && parse_declaration_encoding(data, size, enc, enc_length))
+                {
+                        // iso-8859-1 (case-insensitive)
+                        if (enc_length == 10
+                                && (enc[0] | ' ') == 'i' && (enc[1] | ' ') == 's' && (enc[2] | ' ') == 'o'
+                                && enc[3] == '-' && enc[4] == '8' && enc[5] == '8' && enc[6] == '5' && enc[7] == '9'
+                                && enc[8] == '-' && enc[9] == '1')
+                                return encoding_latin1;
+
+                        // latin1 (case-insensitive)
+                        if (enc_length == 6
+                                && (enc[0] | ' ') == 'l' && (enc[1] | ' ') == 'a' && (enc[2] | ' ') == 't'
+                                && (enc[3] | ' ') == 'i' && (enc[4] | ' ') == 'n'
+                                && enc[5] == '1')
+                                return encoding_latin1;
+                }
+
                 return encoding_utf8;
         }
 
-        xml_encoding get_buffer_encoding(xml_encoding encoding, const void* contents, size_t size)
+        PUGI__FN xml_encoding get_buffer_encoding(xml_encoding encoding, const void* contents, size_t size)
         {
                 // replace wchar encoding with utf implementation
@@ -1041 +2010 @@
                 if (encoding != encoding_auto) return encoding;
 
-                // skip encoding autodetection if input buffer is too small
-                if (size < 4) return encoding_utf8;
-
                 // try to guess encoding (based on XML specification, Appendix F.1)
                 const uint8_t* data = static_cast<const uint8_t*>(contents);
 
-                DMC_VOLATILE uint8_t d0 = data[0], d1 = data[1], d2 = data[2], d3 = data[3];
-
-                return guess_buffer_encoding(d0, d1, d2, d3);
-        }
-
-        bool get_mutable_buffer(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, bool is_mutable)
-        {
+                return guess_buffer_encoding(data, size);
+        }
+
+        PUGI__FN bool get_mutable_buffer(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, bool is_mutable)
+        {
+                size_t length = size / sizeof(char_t);
+
                 if (is_mutable)
                 {
                         out_buffer = static_cast<char_t*>(const_cast<void*>(contents));
+                        out_length = length;
                 }
                 else
                 {
-                        void* buffer = global_allocate(size > 0 ? size : 1);
+                        char_t* buffer = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
                         if (!buffer) return false;
 
-                        memcpy(buffer, contents, size);
-
-                        out_buffer = static_cast<char_t*>(buffer);
-                }
-
-                out_length = size / sizeof(char_t);
+                        if (contents)
+                                memcpy(buffer, contents, length * sizeof(char_t));
+                        else
+                                assert(length == 0);
+
+                        buffer[length] = 0;
+
+                        out_buffer = buffer;
+                        out_length = length + 1;
+                }
 
                 return true;
@@ -1074 +2045 @@
 
 #ifdef PUGIXML_WCHAR_MODE
-        inline bool need_endian_swap_utf(xml_encoding le, xml_encoding re)
+        PUGI__FN bool need_endian_swap_utf(xml_encoding le, xml_encoding re)
         {
                 return (le == encoding_utf16_be && re == encoding_utf16_le) || (le == encoding_utf16_le && re == encoding_utf16_be) ||
-                       (le == encoding_utf32_be && re == encoding_utf32_le) || (le == encoding_utf32_le && re == encoding_utf32_be);
-        }
-
-        bool convert_buffer_endian_swap(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, bool is_mutable)
+                           (le == encoding_utf32_be && re == encoding_utf32_le) || (le == encoding_utf32_le && re == encoding_utf32_be);
+        }
+
+        PUGI__FN bool convert_buffer_endian_swap(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, bool is_mutable)
         {
                 const char_t* data = static_cast<const char_t*>(contents);
-        
+                size_t length = size / sizeof(char_t);
+
                 if (is_mutable)
                 {
-                        out_buffer = const_cast<char_t*>(data);
+                        char_t* buffer = const_cast<char_t*>(data);
+
+                        convert_wchar_endian_swap(buffer, data, length);
+
+                        out_buffer = buffer;
+                        out_length = length;
                 }
                 else
                 {
-                        out_buffer = static_cast<char_t*>(global_allocate(size > 0 ? size : 1));
-                        if (!out_buffer) return false;
-                }
-
-                out_length = size / sizeof(char_t);
-
-                convert_wchar_endian_swap(out_buffer, data, out_length);
+                        char_t* buffer = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
+                        if (!buffer) return false;
+
+                        convert_wchar_endian_swap(buffer, data, length);
+                        buffer[length] = 0;
+
+                        out_buffer = buffer;
+                        out_length = length + 1;
+                }
 
                 return true;
         }
 
-        bool convert_buffer_utf8(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size)
-        {
-                const uint8_t* data = static_cast<const uint8_t*>(contents);
+        template <typename D> PUGI__FN bool convert_buffer_generic(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, D)
+        {
+                const typename D::type* data = static_cast<const typename D::type*>(contents);
+                size_t data_length = size / sizeof(typename D::type);
 
                 // first pass: get length in wchar_t units
-                out_length = utf_decoder<wchar_counter>::decode_utf8_block(data, size, 0);
+                size_t length = D::process(data, data_length, 0, wchar_counter());
 
                 // allocate buffer of suitable length
-                out_buffer = static_cast<char_t*>(global_allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
-                if (!out_buffer) return false;
-
-                // second pass: convert utf8 input to wchar_t
-                wchar_writer::value_type out_begin = reinterpret_cast<wchar_writer::value_type>(out_buffer);
-                wchar_writer::value_type out_end = utf_decoder<wchar_writer>::decode_utf8_block(data, size, out_begin);
-
-                assert(out_end == out_begin + out_length);
-                (void)!out_end;
+                char_t* buffer = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
+                if (!buffer) return false;
+
+                // second pass: convert utf16 input to wchar_t
+                wchar_writer::value_type obegin = reinterpret_cast<wchar_writer::value_type>(buffer);
+                wchar_writer::value_type oend = D::process(data, data_length, obegin, wchar_writer());
+
+                assert(oend == obegin + length);
+                *oend = 0;
+
+                out_buffer = buffer;
+                out_length = length + 1;
 
                 return true;
         }
 
-        template <typename opt_swap> bool convert_buffer_utf16(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
-        {
-                const uint16_t* data = static_cast<const uint16_t*>(contents);
-                size_t length = size / sizeof(uint16_t);
-
-                // first pass: get length in wchar_t units
-                out_length = utf_decoder<wchar_counter, opt_swap>::decode_utf16_block(data, length, 0);
-
-                // allocate buffer of suitable length
-                out_buffer = static_cast<char_t*>(global_allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
-                if (!out_buffer) return false;
-
-                // second pass: convert utf16 input to wchar_t
-                wchar_writer::value_type out_begin = reinterpret_cast<wchar_writer::value_type>(out_buffer);
-                wchar_writer::value_type out_end = utf_decoder<wchar_writer, opt_swap>::decode_utf16_block(data, length, out_begin);
-
-                assert(out_end == out_begin + out_length);
-                (void)!out_end;
-
-                return true;
-        }
-
-        template <typename opt_swap> bool convert_buffer_utf32(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
-        {
-                const uint32_t* data = static_cast<const uint32_t*>(contents);
-                size_t length = size / sizeof(uint32_t);
-
-                // first pass: get length in wchar_t units
-                out_length = utf_decoder<wchar_counter, opt_swap>::decode_utf32_block(data, length, 0);
-
-                // allocate buffer of suitable length
-                out_buffer = static_cast<char_t*>(global_allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
-                if (!out_buffer) return false;
-
-                // second pass: convert utf32 input to wchar_t
-                wchar_writer::value_type out_begin = reinterpret_cast<wchar_writer::value_type>(out_buffer);
-                wchar_writer::value_type out_end = utf_decoder<wchar_writer, opt_swap>::decode_utf32_block(data, length, out_begin);
-
-                assert(out_end == out_begin + out_length);
-                (void)!out_end;
-
-                return true;
-        }
-
-        bool convert_buffer(char_t*& out_buffer, size_t& out_length, xml_encoding encoding, const void* contents, size_t size, bool is_mutable)
+        PUGI__FN bool convert_buffer(char_t*& out_buffer, size_t& out_length, xml_encoding encoding, const void* contents, size_t size, bool is_mutable)
         {
                 // get native encoding
@@ -1172 +2111 @@
 
                 // fast path: no conversion required
-                if (encoding == wchar_encoding) return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
+                if (encoding == wchar_encoding)
+                        return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
 
                 // only endian-swapping is required
-                if (need_endian_swap_utf(encoding, wchar_encoding)) return convert_buffer_endian_swap(out_buffer, out_length, contents, size, is_mutable);
+                if (need_endian_swap_utf(encoding, wchar_encoding))
+                        return convert_buffer_endian_swap(out_buffer, out_length, contents, size, is_mutable);
 
                 // source encoding is utf8
-                if (encoding == encoding_utf8) return convert_buffer_utf8(out_buffer, out_length, contents, size);
+                if (encoding == encoding_utf8)
+                        return convert_buffer_generic(out_buffer, out_length, contents, size, utf8_decoder());
 
                 // source encoding is utf16
@@ -1186 +2128 @@
 
                         return (native_encoding == encoding) ?
-                                convert_buffer_utf16(out_buffer, out_length, contents, size, opt_false()) :
-                                convert_buffer_utf16(out_buffer, out_length, contents, size, opt_true());
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf16_decoder<opt_false>()) :
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf16_decoder<opt_true>());
                 }
 
@@ -1196 +2138 @@
 
                         return (native_encoding == encoding) ?
-                                convert_buffer_utf32(out_buffer, out_length, contents, size, opt_false()) :
-                                convert_buffer_utf32(out_buffer, out_length, contents, size, opt_true());
-                }
-
-                assert(!"Invalid encoding");
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf32_decoder<opt_false>()) :
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf32_decoder<opt_true>());
+                }
+
+                // source encoding is latin1
+                if (encoding == encoding_latin1)
+                        return convert_buffer_generic(out_buffer, out_length, contents, size, latin1_decoder());
+
+                assert(false && "Invalid encoding");
                 return false;
         }
 #else
-        template <typename opt_swap> bool convert_buffer_utf16(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
-        {
-                const uint16_t* data = static_cast<const uint16_t*>(contents);
-                size_t length = size / sizeof(uint16_t);
+        template <typename D> PUGI__FN bool convert_buffer_generic(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, D)
+        {
+                const typename D::type* data = static_cast<const typename D::type*>(contents);
+                size_t data_length = size / sizeof(typename D::type);
 
                 // first pass: get length in utf8 units
-                out_length = utf_decoder<utf8_counter, opt_swap>::decode_utf16_block(data, length, 0);
+                size_t length = D::process(data, data_length, 0, utf8_counter());
 
                 // allocate buffer of suitable length
-                out_buffer = static_cast<char_t*>(global_allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
-                if (!out_buffer) return false;
+                char_t* buffer = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
+                if (!buffer) return false;
 
                 // second pass: convert utf16 input to utf8
-                uint8_t* out_begin = reinterpret_cast<uint8_t*>(out_buffer);
-                uint8_t* out_end = utf_decoder<utf8_writer, opt_swap>::decode_utf16_block(data, length, out_begin);
-
-                assert(out_end == out_begin + out_length);
-                (void)!out_end;
+                uint8_t* obegin = reinterpret_cast<uint8_t*>(buffer);
+                uint8_t* oend = D::process(data, data_length, obegin, utf8_writer());
+
+                assert(oend == obegin + length);
+                *oend = 0;
+
+                out_buffer = buffer;
+                out_length = length + 1;
 
                 return true;
         }
 
-        template <typename opt_swap> bool convert_buffer_utf32(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, opt_swap)
-        {
-                const uint32_t* data = static_cast<const uint32_t*>(contents);
-                size_t length = size / sizeof(uint32_t);
+        PUGI__FN size_t get_latin1_7bit_prefix_length(const uint8_t* data, size_t size)
+        {
+                for (size_t i = 0; i < size; ++i)
+                        if (data[i] > 127)
+                                return i;
+
+                return size;
+        }
+
+        PUGI__FN bool convert_buffer_latin1(char_t*& out_buffer, size_t& out_length, const void* contents, size_t size, bool is_mutable)
+        {
+                const uint8_t* data = static_cast<const uint8_t*>(contents);
+                size_t data_length = size;
+
+                // get size of prefix that does not need utf8 conversion
+                size_t prefix_length = get_latin1_7bit_prefix_length(data, data_length);
+                assert(prefix_length <= data_length);
+
+                const uint8_t* postfix = data + prefix_length;
+                size_t postfix_length = data_length - prefix_length;
+
+                // if no conversion is needed, just return the original buffer
+                if (postfix_length == 0) return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
 
                 // first pass: get length in utf8 units
-                out_length = utf_decoder<utf8_counter, opt_swap>::decode_utf32_block(data, length, 0);
+                size_t length = prefix_length + latin1_decoder::process(postfix, postfix_length, 0, utf8_counter());
 
                 // allocate buffer of suitable length
-                out_buffer = static_cast<char_t*>(global_allocate((out_length > 0 ? out_length : 1) * sizeof(char_t)));
-                if (!out_buffer) return false;
-
-                // second pass: convert utf32 input to utf8
-                uint8_t* out_begin = reinterpret_cast<uint8_t*>(out_buffer);
-                uint8_t* out_end = utf_decoder<utf8_writer, opt_swap>::decode_utf32_block(data, length, out_begin);
-
-                assert(out_end == out_begin + out_length);
-                (void)!out_end;
+                char_t* buffer = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
+                if (!buffer) return false;
+
+                // second pass: convert latin1 input to utf8
+                memcpy(buffer, data, prefix_length);
+
+                uint8_t* obegin = reinterpret_cast<uint8_t*>(buffer);
+                uint8_t* oend = latin1_decoder::process(postfix, postfix_length, obegin + prefix_length, utf8_writer());
+
+                assert(oend == obegin + length);
+                *oend = 0;
+
+                out_buffer = buffer;
+                out_length = length + 1;
 
                 return true;
         }
 
-        bool convert_buffer(char_t*& out_buffer, size_t& out_length, xml_encoding encoding, const void* contents, size_t size, bool is_mutable)
+        PUGI__FN bool convert_buffer(char_t*& out_buffer, size_t& out_length, xml_encoding encoding, const void* contents, size_t size, bool is_mutable)
         {
                 // fast path: no conversion required
-                if (encoding == encoding_utf8) return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
+                if (encoding == encoding_utf8)
+                        return get_mutable_buffer(out_buffer, out_length, contents, size, is_mutable);
 
                 // source encoding is utf16
@@ -1259 +2233 @@
 
                         return (native_encoding == encoding) ?
-                                convert_buffer_utf16(out_buffer, out_length, contents, size, opt_false()) :
-                                convert_buffer_utf16(out_buffer, out_length, contents, size, opt_true());
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf16_decoder<opt_false>()) :
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf16_decoder<opt_true>());
                 }
 
@@ -1269 +2243 @@
 
                         return (native_encoding == encoding) ?
-                                convert_buffer_utf32(out_buffer, out_length, contents, size, opt_false()) :
-                                convert_buffer_utf32(out_buffer, out_length, contents, size, opt_true());
-                }
-
-                assert(!"Invalid encoding");
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf32_decoder<opt_false>()) :
+                                convert_buffer_generic(out_buffer, out_length, contents, size, utf32_decoder<opt_true>());
+                }
+
+                // source encoding is latin1
+                if (encoding == encoding_latin1)
+                        return convert_buffer_latin1(out_buffer, out_length, contents, size, is_mutable);
+
+                assert(false && "Invalid encoding");
                 return false;
         }
 #endif
 
-        size_t as_utf8_begin(const wchar_t* str, size_t length)
-        {
-                STATIC_ASSERT(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4);
-
+        PUGI__FN size_t as_utf8_begin(const wchar_t* str, size_t length)
+        {
                 // get length in utf8 characters
-                return sizeof(wchar_t) == 2 ?
-                        utf_decoder<utf8_counter>::decode_utf16_block(reinterpret_cast<const uint16_t*>(str), length, 0) :
-                        utf_decoder<utf8_counter>::decode_utf32_block(reinterpret_cast<const uint32_t*>(str), length, 0);
-    }
-
-    void as_utf8_end(char* buffer, size_t size, const wchar_t* str, size_t length)
-    {
-                STATIC_ASSERT(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4);
-
-        // convert to utf8
-        uint8_t* begin = reinterpret_cast<uint8_t*>(buffer);
-        uint8_t* end = sizeof(wchar_t) == 2 ?
-            utf_decoder<utf8_writer>::decode_utf16_block(reinterpret_cast<const uint16_t*>(str), length, begin) :
-            utf_decoder<utf8_writer>::decode_utf32_block(reinterpret_cast<const uint32_t*>(str), length, begin);
-    
-        assert(begin + size == end);
-        (void)!end;
-
-                // zero-terminate
-                buffer[size] = 0;
-        }
-    
+                return wchar_decoder::process(str, length, 0, utf8_counter());
+        }
+
+        PUGI__FN void as_utf8_end(char* buffer, size_t size, const wchar_t* str, size_t length)
+        {
+                // convert to utf8
+                uint8_t* begin = reinterpret_cast<uint8_t*>(buffer);
+                uint8_t* end = wchar_decoder::process(str, length, begin, utf8_writer());
+
+                assert(begin + size == end);
+                (void)!end;
+                (void)!size;
+        }
+
 #ifndef PUGIXML_NO_STL
-    std::string as_utf8_impl(const wchar_t* str, size_t length)
-    {
+        PUGI__FN std::string as_utf8_impl(const wchar_t* str, size_t length)
+        {
                 // first pass: get length in utf8 characters
-        size_t size = as_utf8_begin(str, length);
+                size_t size = as_utf8_begin(str, length);
 
                 // allocate resulting string
@@ -1318 +2286 @@
                 if (size > 0) as_utf8_end(&result[0], size, str, length);
 
-                return result;
-    }
-
-        std::wstring as_wide_impl(const char* str, size_t size)
+                return result;
+        }
+
+        PUGI__FN std::basic_string<wchar_t> as_wide_impl(const char* str, size_t size)
         {
                 const uint8_t* data = reinterpret_cast<const uint8_t*>(str);
 
                 // first pass: get length in wchar_t units
-                size_t length = utf_decoder<wchar_counter>::decode_utf8_block(data, size, 0);
+                size_t length = utf8_decoder::process(data, size, 0, wchar_counter());
 
                 // allocate resulting string
-                std::wstring result;
+                std::basic_string<wchar_t> result;
                 result.resize(length);
 
@@ -1336 +2304 @@
                 {
                         wchar_writer::value_type begin = reinterpret_cast<wchar_writer::value_type>(&result[0]);
-                        wchar_writer::value_type end = utf_decoder<wchar_writer>::decode_utf8_block(data, size, begin);
+                        wchar_writer::value_type end = utf8_decoder::process(data, size, begin, wchar_writer());
 
                         assert(begin + length == end);
@@ -1346 +2314 @@
 #endif
 
-        inline bool strcpy_insitu_allow(size_t length, uintptr_t allocated, char_t* target)
-        {
-                assert(target);
+        template <typename Header>
+        inline bool strcpy_insitu_allow(size_t length, const Header& header, uintptr_t header_mask, char_t* target)
+        {
+                // never reuse shared memory
+                if (header & xml_memory_page_contents_shared_mask) return false;
+
                 size_t target_length = strlength(target);
 
                 // always reuse document buffer memory if possible
-                if (!allocated) return target_length >= length;
+                if ((header & header_mask) == 0) return target_length >= length;
 
                 // reuse heap memory if waste is not too great
@@ -1360 +2331 @@
         }
 
-        bool strcpy_insitu(char_t*& dest, uintptr_t& header, uintptr_t header_mask, const char_t* source)
-        {
-                size_t source_length = strlength(source);
-
+        template <typename String, typename Header>
+        PUGI__FN bool strcpy_insitu(String& dest, Header& header, uintptr_t header_mask, const char_t* source, size_t source_length)
+        {
                 if (source_length == 0)
                 {
                         // empty string and null pointer are equivalent, so just deallocate old memory
-                        xml_allocator* alloc = reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask)->allocator;
+                        xml_allocator* alloc = PUGI__GETPAGE_IMPL(header)->allocator;
 
                         if (header & header_mask) alloc->deallocate_string(dest);
-                        
+
                         // mark the string as not allocated
                         dest = 0;
@@ -1377 +2347 @@
                         return true;
                 }
-                else if (dest && strcpy_insitu_allow(source_length, header & header_mask, dest))
+                else if (dest && strcpy_insitu_allow(source_length, header, header_mask, dest))
                 {
                         // we can reuse old buffer, so just copy the new data (including zero terminator)
-                        memcpy(dest, source, (source_length + 1) * sizeof(char_t));
-                        
+                        memcpy(dest, source, source_length * sizeof(char_t));
+                        dest[source_length] = 0;
+
                         return true;
                 }
                 else
                 {
-                        xml_allocator* alloc = reinterpret_cast<xml_memory_page*>(header & xml_memory_page_pointer_mask)->allocator;
+                        xml_allocator* alloc = PUGI__GETPAGE_IMPL(header)->allocator;
+
+                        if (!alloc->reserve()) return false;
 
                         // allocate new buffer
@@ -1393 +2366 @@
 
                         // copy the string (including zero terminator)
-                        memcpy(buf, source, (source_length + 1) * sizeof(char_t));
+                        memcpy(buf, source, source_length * sizeof(char_t));
+                        buf[source_length] = 0;
 
                         // deallocate old buffer (*after* the above to protect against overlapping memory and/or allocation failures)
                         if (header & header_mask) alloc->deallocate_string(dest);
-                        
+
                         // the string is now allocated, so set the flag
                         dest = buf;
@@ -1410 +2384 @@
                 char_t* end;
                 size_t size;
-                        
+
                 gap(): end(0), size(0)
                 {
                 }
-                        
+
                 // Push new gap, move s count bytes further (skipping the gap).
                 // Collapse previous gap.
@@ -1425 +2399 @@
                                 memmove(end - size, end, reinterpret_cast<char*>(s) - reinterpret_cast<char*>(end));
                         }
-                                
+
                         s += count; // end of current gap
-                                
+
                         // "merge" two gaps
                         end = s;
                         size += count;
                 }
-                        
+
                 // Collapse all gaps, return past-the-end pointer
                 char_t* flush(char_t* s)
@@ -1447 +2421 @@
                 }
         };
-        
-        char_t* strconv_escape(char_t* s, gap& g)
+
+        PUGI__FN char_t* strconv_escape(char_t* s, gap& g)
         {
                 char_t* stre = s + 1;
@@ -1479 +2453 @@
                                                 ch = *++stre;
                                         }
-                                        
+
                                         ++stre;
                                 }
@@ -1490 +2464 @@
                                         for (;;)
                                         {
-                                                if (static_cast<unsigned int>(ch - '0') <= 9)
+                                                if (static_cast<unsigned int>(static_cast<unsigned int>(ch) - '0') <= 9)
                                                         ucsc = 10 * ucsc + (ch - '0');
                                                 else if (ch == ';')
@@ -1499 +2473 @@
                                                 ch = *++stre;
                                         }
-                                        
+
                                         ++stre;
                                 }
@@ -1508 +2482 @@
                                 s = reinterpret_cast<char_t*>(utf8_writer::any(reinterpret_cast<uint8_t*>(s), ucsc));
                         #endif
-                                        
+
                                 g.push(s, stre - s);
                                 return stre;
                         }
+
                         case 'a':       // &a
                         {
@@ -1522 +2497 @@
                                                 *s++ = '&';
                                                 ++stre;
-                                                        
+
                                                 g.push(s, stre - s);
                                                 return stre;
@@ -1540 +2515 @@
                                 break;
                         }
+
                         case 'g': // &g
                         {
@@ -1546 +2522 @@
                                         *s++ = '>';
                                         ++stre;
-                                        
+
                                         g.push(s, stre - s);
                                         return stre;
@@ -1552 +2528 @@
                                 break;
                         }
+
                         case 'l': // &l
                         {
@@ -1558 +2535 @@
                                         *s++ = '<';
                                         ++stre;
-                                                
+
                                         g.push(s, stre - s);
                                         return stre;
@@ -1564 +2541 @@
                                 break;
                         }
+
                         case 'q': // &q
                         {
@@ -1570 +2548 @@
                                         *s++ = '"';
                                         ++stre;
-                                        
+
                                         g.push(s, stre - s);
                                         return stre;
@@ -1576 +2554 @@
                                 break;
                         }
-                }
-                
+
+                        default:
+                                break;
+                }
+
                 return stre;
         }
 
-        // Utility macro for last character handling
-        #define ENDSWITH(c, e) ((c) == (e) || ((c) == 0 && endch == (e)))
-
-        char_t* strconv_comment(char_t* s, char_t endch)
+        // Parser utilities
+        #define PUGI__ENDSWITH(c, e)        ((c) == (e) || ((c) == 0 && endch == (e)))
+        #define PUGI__SKIPWS()              { while (PUGI__IS_CHARTYPE(*s, ct_space)) ++s; }
+        #define PUGI__OPTSET(OPT)           ( optmsk & (OPT) )
+        #define PUGI__PUSHNODE(TYPE)        { cursor = append_new_node(cursor, *alloc, TYPE); if (!cursor) PUGI__THROW_ERROR(status_out_of_memory, s); }
+        #define PUGI__POPNODE()             { cursor = cursor->parent; }
+        #define PUGI__SCANFOR(X)            { while (*s != 0 && !(X)) ++s; }
+        #define PUGI__SCANWHILE(X)          { while (X) ++s; }
+        #define PUGI__SCANWHILE_UNROLL(X)   { for (;;) { char_t ss = s[0]; if (PUGI__UNLIKELY(!(X))) { break; } ss = s[1]; if (PUGI__UNLIKELY(!(X))) { s += 1; break; } ss = s[2]; if (PUGI__UNLIKELY(!(X))) { s += 2; break; } ss = s[3]; if (PUGI__UNLIKELY(!(X))) { s += 3; break; } s += 4; } }
+        #define PUGI__ENDSEG()              { ch = *s; *s = 0; ++s; }
+        #define PUGI__THROW_ERROR(err, m)   return error_offset = m, error_status = err, static_cast<char_t*>(0)
+        #define PUGI__CHECK_ERROR(err, m)   { if (*s == 0) PUGI__THROW_ERROR(err, m); }
+
+        PUGI__FN char_t* strconv_comment(char_t* s, char_t endch)
         {
                 gap g;
-                
+
                 while (true)
                 {
-                        while (!IS_CHARTYPE(*s, ct_parse_comment)) ++s;
-                
+                        PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_comment));
+
                         if (*s == '\r') // Either a single 0x0d or 0x0d 0x0a pair
                         {
                                 *s++ = '\n'; // replace first one with 0x0a
-                                
+
                                 if (*s == '\n') g.push(s, 1);
                         }
-                        else if (s[0] == '-' && s[1] == '-' && ENDSWITH(s[2], '>')) // comment ends here
+                        else if (s[0] == '-' && s[1] == '-' && PUGI__ENDSWITH(s[2], '>')) // comment ends here
                         {
                                 *g.flush(s) = 0;
-                                
+
                                 return s + (s[2] == '>' ? 3 : 2);
                         }
@@ -1612 +2603 @@
         }
 
-        char_t* strconv_cdata(char_t* s, char_t endch)
+        PUGI__FN char_t* strconv_cdata(char_t* s, char_t endch)
         {
                 gap g;
-                        
+
                 while (true)
                 {
-                        while (!IS_CHARTYPE(*s, ct_parse_cdata)) ++s;
-                        
+                        PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_cdata));
+
                         if (*s == '\r') // Either a single 0x0d or 0x0d 0x0a pair
                         {
                                 *s++ = '\n'; // replace first one with 0x0a
-                                
+
                                 if (*s == '\n') g.push(s, 1);
                         }
-                        else if (s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>')) // CDATA ends here
+                        else if (s[0] == ']' && s[1] == ']' && PUGI__ENDSWITH(s[2], '>')) // CDATA ends here
                         {
                                 *g.flush(s) = 0;
-                                
+
                                 return s + 1;
                         }
@@ -1639 +2630 @@
                 }
         }
-        
+
         typedef char_t* (*strconv_pcdata_t)(char_t*);
-                
-        template <typename opt_eol, typename opt_escape> struct strconv_pcdata_impl
+
+        template <typename opt_trim, typename opt_eol, typename opt_escape> struct strconv_pcdata_impl
         {
                 static char_t* parse(char_t* s)
                 {
                         gap g;
-                        
+
+                        char_t* begin = s;
+
                         while (true)
                         {
-                                while (!IS_CHARTYPE(*s, ct_parse_pcdata)) ++s;
-                                        
+                                PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_pcdata));
+
                                 if (*s == '<') // PCDATA ends here
                                 {
-                                        *g.flush(s) = 0;
-                                        
+                                        char_t* end = g.flush(s);
+
+                                        if (opt_trim::value)
+                                                while (end > begin && PUGI__IS_CHARTYPE(end[-1], ct_space))
+                                                        --end;
+
+                                        *end = 0;
+
                                         return s + 1;
                                 }
@@ -1661 +2660 @@
                                 {
                                         *s++ = '\n'; // replace first one with 0x0a
-                                        
+
                                         if (*s == '\n') g.push(s, 1);
                                 }
@@ -1670 +2669 @@
                                 else if (*s == 0)
                                 {
+                                        char_t* end = g.flush(s);
+
+                                        if (opt_trim::value)
+                                                while (end > begin && PUGI__IS_CHARTYPE(end[-1], ct_space))
+                                                        --end;
+
+                                        *end = 0;
+
                                         return s;
                                 }
@@ -1676 +2683 @@
                 }
         };
-        
-        strconv_pcdata_t get_strconv_pcdata(unsigned int optmask)
-        {
-                STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20);
-
-                switch ((optmask >> 4) & 3) // get bitmask for flags (eol escapes)
-                {
-                case 0: return strconv_pcdata_impl<opt_false, opt_false>::parse;
-                case 1: return strconv_pcdata_impl<opt_false, opt_true>::parse;
-                case 2: return strconv_pcdata_impl<opt_true, opt_false>::parse;
-                case 3: return strconv_pcdata_impl<opt_true, opt_true>::parse;
-                default: return 0; // should not get here
+
+        PUGI__FN strconv_pcdata_t get_strconv_pcdata(unsigned int optmask)
+        {
+                PUGI__STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20 && parse_trim_pcdata == 0x0800);
+
+                switch (((optmask >> 4) & 3) | ((optmask >> 9) & 4)) // get bitmask for flags (eol escapes trim)
+                {
+                case 0: return strconv_pcdata_impl<opt_false, opt_false, opt_false>::parse;
+                case 1: return strconv_pcdata_impl<opt_false, opt_false, opt_true>::parse;
+                case 2: return strconv_pcdata_impl<opt_false, opt_true, opt_false>::parse;
+                case 3: return strconv_pcdata_impl<opt_false, opt_true, opt_true>::parse;
+                case 4: return strconv_pcdata_impl<opt_true, opt_false, opt_false>::parse;
+                case 5: return strconv_pcdata_impl<opt_true, opt_false, opt_true>::parse;
+                case 6: return strconv_pcdata_impl<opt_true, opt_true, opt_false>::parse;
+                case 7: return strconv_pcdata_impl<opt_true, opt_true, opt_true>::parse;
+                default: assert(false); return 0; // should not get here
                 }
         }
 
         typedef char_t* (*strconv_attribute_t)(char_t*, char_t);
-        
+
         template <typename opt_escape> struct strconv_attribute_impl
         {
@@ -1700 +2711 @@
 
                         // trim leading whitespaces
-                        if (IS_CHARTYPE(*s, ct_space))
+                        if (PUGI__IS_CHARTYPE(*s, ct_space))
                         {
                                 char_t* str = s;
-                                
+
                                 do ++str;
-                                while (IS_CHARTYPE(*str, ct_space));
-                                
+                                while (PUGI__IS_CHARTYPE(*str, ct_space));
+
                                 g.push(s, str - s);
                         }
@@ -1712 +2723 @@
                         while (true)
                         {
-                                while (!IS_CHARTYPE(*s, ct_parse_attr_ws | ct_space)) ++s;
-                                
+                                PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_attr_ws | ct_space));
+
                                 if (*s == end_quote)
                                 {
                                         char_t* str = g.flush(s);
-                                        
+
                                         do *str-- = 0;
-                                        while (IS_CHARTYPE(*str, ct_space));
-                                
+                                        while (PUGI__IS_CHARTYPE(*str, ct_space));
+
                                         return s + 1;
                                 }
-                                else if (IS_CHARTYPE(*s, ct_space))
+                                else if (PUGI__IS_CHARTYPE(*s, ct_space))
                                 {
                                         *s++ = ' ';
-                
-                                        if (IS_CHARTYPE(*s, ct_space))
+
+                                        if (PUGI__IS_CHARTYPE(*s, ct_space))
                                         {
                                                 char_t* str = s + 1;
-                                                while (IS_CHARTYPE(*str, ct_space)) ++str;
-                                                
+                                                while (PUGI__IS_CHARTYPE(*str, ct_space)) ++str;
+
                                                 g.push(s, str - s);
                                         }
@@ -1753 +2764 @@
                         while (true)
                         {
-                                while (!IS_CHARTYPE(*s, ct_parse_attr_ws)) ++s;
-                                
+                                PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_attr_ws));
+
                                 if (*s == end_quote)
                                 {
                                         *g.flush(s) = 0;
-                                
+
                                         return s + 1;
                                 }
-                                else if (IS_CHARTYPE(*s, ct_space))
+                                else if (PUGI__IS_CHARTYPE(*s, ct_space))
                                 {
                                         if (*s == '\r')
                                         {
                                                 *s++ = ' ';
-                                
+
                                                 if (*s == '\n') g.push(s, 1);
                                         }
@@ -1789 +2800 @@
                         while (true)
                         {
-                                while (!IS_CHARTYPE(*s, ct_parse_attr)) ++s;
-                                
+                                PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_attr));
+
                                 if (*s == end_quote)
                                 {
                                         *g.flush(s) = 0;
-                                
+
                                         return s + 1;
                                 }
@@ -1800 +2811 @@
                                 {
                                         *s++ = '\n';
-                                        
+
                                         if (*s == '\n') g.push(s, 1);
                                 }
@@ -1821 +2832 @@
                         while (true)
                         {
-                                while (!IS_CHARTYPE(*s, ct_parse_attr)) ++s;
-                                
+                                PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPE(ss, ct_parse_attr));
+
                                 if (*s == end_quote)
                                 {
                                         *g.flush(s) = 0;
-                                
+
                                         return s + 1;
                                 }
@@ -1842 +2853 @@
         };
 
-        strconv_attribute_t get_strconv_attribute(unsigned int optmask)
-        {
-                STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20 && parse_wconv_attribute == 0x40 && parse_wnorm_attribute == 0x80);
-                
+        PUGI__FN strconv_attribute_t get_strconv_attribute(unsigned int optmask)
+        {
+                PUGI__STATIC_ASSERT(parse_escapes == 0x10 && parse_eol == 0x20 && parse_wconv_attribute == 0x40 && parse_wnorm_attribute == 0x80);
+
                 switch ((optmask >> 4) & 15) // get bitmask for flags (wconv wnorm eol escapes)
                 {
@@ -1864 +2875 @@
                 case 14: return strconv_attribute_impl<opt_false>::parse_wnorm;
                 case 15: return strconv_attribute_impl<opt_true>::parse_wnorm;
-                default: return 0; // should not get here
+                default: assert(false); return 0; // should not get here
                 }
         }
@@ -1879 +2890 @@
         struct xml_parser
         {
-                xml_allocator alloc;
+                xml_allocator* alloc;
                 char_t* error_offset;
-                jmp_buf error_handler;
-                
-                // Parser utilities.
-                #define SKIPWS()                        { while (IS_CHARTYPE(*s, ct_space)) ++s; }
-                #define OPTSET(OPT)                     ( optmsk & OPT )
-                #define PUSHNODE(TYPE)          { cursor = append_node(cursor, alloc, TYPE); if (!cursor) THROW_ERROR(status_out_of_memory, s); }
-                #define POPNODE()                       { cursor = cursor->parent; }
-                #define SCANFOR(X)                      { while (*s != 0 && !(X)) ++s; }
-                #define SCANWHILE(X)            { while ((X)) ++s; }
-                #define ENDSEG()                        { ch = *s; *s = 0; ++s; }
-                #define THROW_ERROR(err, m)     error_offset = m, longjmp(error_handler, err)
-                #define CHECK_ERROR(err, m)     { if (*s == 0) THROW_ERROR(err, m); }
-                
-                xml_parser(const xml_allocator& alloc): alloc(alloc), error_offset(0)
+                xml_parse_status error_status;
+
+                xml_parser(xml_allocator* alloc_): alloc(alloc_), error_offset(0), error_status(status_ok)
                 {
                 }
@@ -1911 +2911 @@
                                 // quoted string
                                 char_t ch = *s++;
-                                SCANFOR(*s == ch);
-                                if (!*s) THROW_ERROR(status_bad_doctype, s);
+                                PUGI__SCANFOR(*s == ch);
+                                if (!*s) PUGI__THROW_ERROR(status_bad_doctype, s);
 
                                 s++;
@@ -1920 +2920 @@
                                 // <? ... ?>
                                 s += 2;
-                                SCANFOR(s[0] == '?' && s[1] == '>'); // no need for ENDSWITH because ?> can't terminate proper doctype
-                                if (!*s) THROW_ERROR(status_bad_doctype, s);
+                                PUGI__SCANFOR(s[0] == '?' && s[1] == '>'); // no need for ENDSWITH because ?> can't terminate proper doctype
+                                if (!*s) PUGI__THROW_ERROR(status_bad_doctype, s);
 
                                 s += 2;
@@ -1928 +2928 @@
                         {
                                 s += 4;
-                                SCANFOR(s[0] == '-' && s[1] == '-' && s[2] == '>'); // no need for ENDSWITH because --> can't terminate proper doctype
-                                if (!*s) THROW_ERROR(status_bad_doctype, s);
-
-                                s += 4;
-                        }
-                        else THROW_ERROR(status_bad_doctype, s);
+                                PUGI__SCANFOR(s[0] == '-' && s[1] == '-' && s[2] == '>'); // no need for ENDSWITH because --> can't terminate proper doctype
+                                if (!*s) PUGI__THROW_ERROR(status_bad_doctype, s);
+
+                                s += 3;
+                        }
+                        else PUGI__THROW_ERROR(status_bad_doctype, s);
 
                         return s;
@@ -1940 +2940 @@
                 char_t* parse_doctype_ignore(char_t* s)
                 {
+                        size_t depth = 0;
+
                         assert(s[0] == '<' && s[1] == '!' && s[2] == '[');
-                        s++;
+                        s += 3;
 
                         while (*s)
@@ -1948 +2950 @@
                                 {
                                         // nested ignore section
-                                        s = parse_doctype_ignore(s);
+                                        s += 3;
+                                        depth++;
                                 }
                                 else if (s[0] == ']' && s[1] == ']' && s[2] == '>')
@@ -1955 +2958 @@
                                         s += 3;
 
-                                        return s;
+                                        if (depth == 0)
+                                                return s;
+
+                                        depth--;
                                 }
                                 else s++;
                         }
 
-                        THROW_ERROR(status_bad_doctype, s);
-
-                        return s;
-                }
-
-                char_t* parse_doctype_group(char_t* s, char_t endch, bool toplevel)
-                {
-                        assert(s[0] == '<' && s[1] == '!');
-                        s++;
+                        PUGI__THROW_ERROR(status_bad_doctype, s);
+                }
+
+                char_t* parse_doctype_group(char_t* s, char_t endch)
+                {
+                        size_t depth = 0;
+
+                        assert((s[0] == '<' || s[0] == 0) && s[1] == '!');
+                        s += 2;
 
                         while (*s)
@@ -1978 +2984 @@
                                                 // ignore
                                                 s = parse_doctype_ignore(s);
+                                                if (!s) return s;
                                         }
                                         else
                                         {
                                                 // some control group
-                                                s = parse_doctype_group(s, endch, false);
+                                                s += 2;
+                                                depth++;
                                         }
                                 }
@@ -1989 +2997 @@
                                         // unknown tag (forbidden), or some primitive group
                                         s = parse_doctype_primitive(s);
+                                        if (!s) return s;
                                 }
                                 else if (*s == '>')
                                 {
+                                        if (depth == 0)
+                                                return s;
+
+                                        depth--;
                                         s++;
-
-                                        return s;
                                 }
                                 else s++;
                         }
 
-                        if (!toplevel || endch != '>') THROW_ERROR(status_bad_doctype, s);
+                        if (depth != 0 || endch != '>') PUGI__THROW_ERROR(status_bad_doctype, s);
 
                         return s;
@@ -2017 +3028 @@
                                         ++s;
 
-                                        if (OPTSET(parse_comments))
+                                        if (PUGI__OPTSET(parse_comments))
                                         {
-                                                PUSHNODE(node_comment); // Append a new node on the tree.
+                                                PUGI__PUSHNODE(node_comment); // Append a new node on the tree.
                                                 cursor->value = s; // Save the offset.
                                         }
 
-                                        if (OPTSET(parse_eol) && OPTSET(parse_comments))
+                                        if (PUGI__OPTSET(parse_eol) && PUGI__OPTSET(parse_comments))
                                         {
                                                 s = strconv_comment(s, endch);
 
-                                                if (!s) THROW_ERROR(status_bad_comment, cursor->value);
+                                                if (!s) PUGI__THROW_ERROR(status_bad_comment, cursor->value);
                                         }
                                         else
                                         {
                                                 // Scan for terminating '-->'.
-                                                SCANFOR(s[0] == '-' && s[1] == '-' && ENDSWITH(s[2], '>'));
-                                                CHECK_ERROR(status_bad_comment, s);
-
-                                                if (OPTSET(parse_comments))
+                                                PUGI__SCANFOR(s[0] == '-' && s[1] == '-' && PUGI__ENDSWITH(s[2], '>'));
+                                                PUGI__CHECK_ERROR(status_bad_comment, s);
+
+                                                if (PUGI__OPTSET(parse_comments))
                                                         *s = 0; // Zero-terminate this segment at the first terminating '-'.
 
@@ -2041 +3052 @@
                                         }
                                 }
-                                else THROW_ERROR(status_bad_comment, s);
+                                else PUGI__THROW_ERROR(status_bad_comment, s);
                         }
                         else if (*s == '[')
@@ -2050 +3061 @@
                                         ++s;
 
-                                        if (OPTSET(parse_cdata))
+                                        if (PUGI__OPTSET(parse_cdata))
                                         {
-                                                PUSHNODE(node_cdata); // Append a new node on the tree.
+                                                PUGI__PUSHNODE(node_cdata); // Append a new node on the tree.
                                                 cursor->value = s; // Save the offset.
 
-                                                if (OPTSET(parse_eol))
+                                                if (PUGI__OPTSET(parse_eol))
                                                 {
                                                         s = strconv_cdata(s, endch);
 
-                                                        if (!s) THROW_ERROR(status_bad_cdata, cursor->value);
+                                                        if (!s) PUGI__THROW_ERROR(status_bad_cdata, cursor->value);
                                                 }
                                                 else
                                                 {
                                                         // Scan for terminating ']]>'.
-                                                        SCANFOR(s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>'));
-                                                        CHECK_ERROR(status_bad_cdata, s);
+                                                        PUGI__SCANFOR(s[0] == ']' && s[1] == ']' && PUGI__ENDSWITH(s[2], '>'));
+                                                        PUGI__CHECK_ERROR(status_bad_cdata, s);
 
                                                         *s++ = 0; // Zero-terminate this segment.
@@ -2073 +3084 @@
                                         {
                                                 // Scan for terminating ']]>'.
-                                                SCANFOR(s[0] == ']' && s[1] == ']' && ENDSWITH(s[2], '>'));
-                                                CHECK_ERROR(status_bad_cdata, s);
+                                                PUGI__SCANFOR(s[0] == ']' && s[1] == ']' && PUGI__ENDSWITH(s[2], '>'));
+                                                PUGI__CHECK_ERROR(status_bad_cdata, s);
 
                                                 ++s;
@@ -2081 +3092 @@
                                         s += (s[1] == '>' ? 2 : 1); // Step over the last ']>'.
                                 }
-                                else THROW_ERROR(status_bad_cdata, s);
-                        }
-                        else if (s[0] == 'D' && s[1] == 'O' && s[2] == 'C' && s[3] == 'T' && s[4] == 'Y' && s[5] == 'P' && ENDSWITH(s[6], 'E'))
+                                else PUGI__THROW_ERROR(status_bad_cdata, s);
+                        }
+                        else if (s[0] == 'D' && s[1] == 'O' && s[2] == 'C' && s[3] == 'T' && s[4] == 'Y' && s[5] == 'P' && PUGI__ENDSWITH(s[6], 'E'))
                         {
                                 s -= 2;
 
-                if (cursor->parent) THROW_ERROR(status_bad_doctype, s);
-
-                char_t* mark = s + 9;
-
-                                s = parse_doctype_group(s, endch, true);
-
-                if (OPTSET(parse_doctype))
-                {
-                    while (IS_CHARTYPE(*mark, ct_space)) ++mark;
-
-                    PUSHNODE(node_doctype);
-
-                    cursor->value = mark;
-
-                    assert((s[0] == 0 && endch == '>') || s[-1] == '>');
-                    s[*s == 0 ? 0 : -1] = 0;
-
-                    POPNODE();
-                }
-                        }
-                        else if (*s == 0 && endch == '-') THROW_ERROR(status_bad_comment, s);
-                        else if (*s == 0 && endch == '[') THROW_ERROR(status_bad_cdata, s);
-                        else THROW_ERROR(status_unrecognized_tag, s);
+                                if (cursor->parent) PUGI__THROW_ERROR(status_bad_doctype, s);
+
+                                char_t* mark = s + 9;
+
+                                s = parse_doctype_group(s, endch);
+                                if (!s) return s;
+
+                                assert((*s == 0 && endch == '>') || *s == '>');
+                                if (*s) *s++ = 0;
+
+                                if (PUGI__OPTSET(parse_doctype))
+                                {
+                                        while (PUGI__IS_CHARTYPE(*mark, ct_space)) ++mark;
+
+                                        PUGI__PUSHNODE(node_doctype);
+
+                                        cursor->value = mark;
+                                }
+                        }
+                        else if (*s == 0 && endch == '-') PUGI__THROW_ERROR(status_bad_comment, s);
+                        else if (*s == 0 && endch == '[') PUGI__THROW_ERROR(status_bad_cdata, s);
+                        else PUGI__THROW_ERROR(status_unrecognized_tag, s);
 
                         return s;
@@ -2126 +3136 @@
                         char_t* target = s;
 
-                        if (!IS_CHARTYPE(*s, ct_start_symbol)) THROW_ERROR(status_bad_pi, s);
-
-                        SCANWHILE(IS_CHARTYPE(*s, ct_symbol));
-                        CHECK_ERROR(status_bad_pi, s);
+                        if (!PUGI__IS_CHARTYPE(*s, ct_start_symbol)) PUGI__THROW_ERROR(status_bad_pi, s);
+
+                        PUGI__SCANWHILE(PUGI__IS_CHARTYPE(*s, ct_symbol));
+                        PUGI__CHECK_ERROR(status_bad_pi, s);
 
                         // determine node type; stricmp / strcasecmp is not portable
                         bool declaration = (target[0] | ' ') == 'x' && (target[1] | ' ') == 'm' && (target[2] | ' ') == 'l' && target + 3 == s;
 
-                        if (declaration ? OPTSET(parse_declaration) : OPTSET(parse_pi))
+                        if (declaration ? PUGI__OPTSET(parse_declaration) : PUGI__OPTSET(parse_pi))
                         {
                                 if (declaration)
                                 {
                                         // disallow non top-level declarations
-                                        if (cursor->parent) THROW_ERROR(status_bad_pi, s);
-
-                                        PUSHNODE(node_declaration);
+                                        if (cursor->parent) PUGI__THROW_ERROR(status_bad_pi, s);
+
+                                        PUGI__PUSHNODE(node_declaration);
                                 }
                                 else
                                 {
-                                        PUSHNODE(node_pi);
+                                        PUGI__PUSHNODE(node_pi);
                                 }
 
                                 cursor->name = target;
 
-                                ENDSEG();
+                                PUGI__ENDSEG();
 
                                 // parse value/attributes
@@ -2156 +3166 @@
                                 {
                                         // empty node
-                                        if (!ENDSWITH(*s, '>')) THROW_ERROR(status_bad_pi, s);
+                                        if (!PUGI__ENDSWITH(*s, '>')) PUGI__THROW_ERROR(status_bad_pi, s);
                                         s += (*s == '>');
 
-                                        POPNODE();
-                                }
-                                else if (IS_CHARTYPE(ch, ct_space))
-                                {
-                                        SKIPWS();
+                                        PUGI__POPNODE();
+                                }
+                                else if (PUGI__IS_CHARTYPE(ch, ct_space))
+                                {
+                                        PUGI__SKIPWS();
 
                                         // scan for tag end
                                         char_t* value = s;
 
-                                        SCANFOR(s[0] == '?' && ENDSWITH(s[1], '>'));
-                                        CHECK_ERROR(status_bad_pi, s);
+                                        PUGI__SCANFOR(s[0] == '?' && PUGI__ENDSWITH(s[1], '>'));
+                                        PUGI__CHECK_ERROR(status_bad_pi, s);
 
                                         if (declaration)
@@ -2183 +3193 @@
                                                 // store value and step over >
                                                 cursor->value = value;
-                                                POPNODE();
-
-                                                ENDSEG();
+
+                                                PUGI__POPNODE();
+
+                                                PUGI__ENDSEG();
 
                                                 s += (*s == '>');
                                         }
                                 }
-                                else THROW_ERROR(status_bad_pi, s);
+                                else PUGI__THROW_ERROR(status_bad_pi, s);
                         }
                         else
                         {
                                 // scan for tag end
-                                SCANFOR(s[0] == '?' && ENDSWITH(s[1], '>'));
-                                CHECK_ERROR(status_bad_pi, s);
+                                PUGI__SCANFOR(s[0] == '?' && PUGI__ENDSWITH(s[1], '>'));
+                                PUGI__CHECK_ERROR(status_bad_pi, s);
 
                                 s += (s[1] == '>' ? 2 : 1);
@@ -2207 +3218 @@
                 }
 
-                void parse(char_t* s, xml_node_struct* xmldoc, unsigned int optmsk, char_t endch)
+                char_t* parse_tree(char_t* s, xml_node_struct* root, unsigned int optmsk, char_t endch)
                 {
                         strconv_attribute_t strconv_attribute = get_strconv_attribute(optmsk);
                         strconv_pcdata_t strconv_pcdata = get_strconv_pcdata(optmsk);
-                        
+
                         char_t ch = 0;
-                        xml_node_struct* cursor = xmldoc;
+                        xml_node_struct* cursor = root;
                         char_t* mark = s;
 
@@ -2223 +3234 @@
 
                                 LOC_TAG:
-                                        if (IS_CHARTYPE(*s, ct_start_symbol)) // '<#...'
+                                        if (PUGI__IS_CHARTYPE(*s, ct_start_symbol)) // '<#...'
                                         {
-                                                PUSHNODE(node_element); // Append a new node to the tree.
+                                                PUGI__PUSHNODE(node_element); // Append a new node to the tree.
 
                                                 cursor->name = s;
 
-                                                SCANWHILE(IS_CHARTYPE(*s, ct_symbol)); // Scan for a terminator.
-                                                ENDSEG(); // Save char in 'ch', terminate & step over.
+                                                PUGI__SCANWHILE_UNROLL(PUGI__IS_CHARTYPE(ss, ct_symbol)); // Scan for a terminator.
+                                                PUGI__ENDSEG(); // Save char in 'ch', terminate & step over.
 
                                                 if (ch == '>')
@@ -2236 +3247 @@
                                                         // end of tag
                                                 }
-                                                else if (IS_CHARTYPE(ch, ct_space))
+                                                else if (PUGI__IS_CHARTYPE(ch, ct_space))
                                                 {
                                                 LOC_ATTRIBUTES:
-                                                    while (true)
-                                                    {
-                                                                SKIPWS(); // Eat any whitespace.
-                                                
-                                                                if (IS_CHARTYPE(*s, ct_start_symbol)) // <... #...
+                                                        while (true)
+                                                        {
+                                                                PUGI__SKIPWS(); // Eat any whitespace.
+
+                                                                if (PUGI__IS_CHARTYPE(*s, ct_start_symbol)) // <... #...
                                                                 {
-                                                                        xml_attribute_struct* a = append_attribute_ll(cursor, alloc); // Make space for this attribute.
-                                                                        if (!a) THROW_ERROR(status_out_of_memory, s);
+                                                                        xml_attribute_struct* a = append_new_attribute(cursor, *alloc); // Make space for this attribute.
+                                                                        if (!a) PUGI__THROW_ERROR(status_out_of_memory, s);
 
                                                                         a->name = s; // Save the offset.
 
-                                                                        SCANWHILE(IS_CHARTYPE(*s, ct_symbol)); // Scan for a terminator.
-                                                                        CHECK_ERROR(status_bad_attribute, s); //$ redundant, left for performance
-
-                                                                        ENDSEG(); // Save char in 'ch', terminate & step over.
-                                                                        CHECK_ERROR(status_bad_attribute, s); //$ redundant, left for performance
-
-                                                                        if (IS_CHARTYPE(ch, ct_space))
+                                                                        PUGI__SCANWHILE_UNROLL(PUGI__IS_CHARTYPE(ss, ct_symbol)); // Scan for a terminator.
+                                                                        PUGI__ENDSEG(); // Save char in 'ch', terminate & step over.
+
+                                                                        if (PUGI__IS_CHARTYPE(ch, ct_space))
                                                                         {
-                                                                                SKIPWS(); // Eat any whitespace.
-                                                                                CHECK_ERROR(status_bad_attribute, s); //$ redundant, left for performance
+                                                                                PUGI__SKIPWS(); // Eat any whitespace.
 
                                                                                 ch = *s;
                                                                                 ++s;
                                                                         }
-                                                                        
+
                                                                         if (ch == '=') // '<... #=...'
                                                                         {
-                                                                                SKIPWS(); // Eat any whitespace.
+                                                                                PUGI__SKIPWS(); // Eat any whitespace.
 
                                                                                 if (*s == '"' || *s == '\'') // '<... #="...'
@@ -2276 +3283 @@
 
                                                                                         s = strconv_attribute(s, ch);
-                                                                                
-                                                                                        if (!s) THROW_ERROR(status_bad_attribute, a->value);
+
+                                                                                        if (!s) PUGI__THROW_ERROR(status_bad_attribute, a->value);
 
                                                                                         // After this line the loop continues from the start;
                                                                                         // Whitespaces, / and > are ok, symbols and EOF are wrong,
                                                                                         // everything else will be detected
-                                                                                        if (IS_CHARTYPE(*s, ct_start_symbol)) THROW_ERROR(status_bad_attribute, s);
+                                                                                        if (PUGI__IS_CHARTYPE(*s, ct_start_symbol)) PUGI__THROW_ERROR(status_bad_attribute, s);
                                                                                 }
-                                                                                else THROW_ERROR(status_bad_attribute, s);
+                                                                                else PUGI__THROW_ERROR(status_bad_attribute, s);
                                                                         }
-                                                                        else THROW_ERROR(status_bad_attribute, s);
+                                                                        else PUGI__THROW_ERROR(status_bad_attribute, s);
                                                                 }
                                                                 else if (*s == '/')
                                                                 {
                                                                         ++s;
-                                                                        
+
                                                                         if (*s == '>')
                                                                         {
-                                                                                POPNODE();
+                                                                                PUGI__POPNODE();
                                                                                 s++;
                                                                                 break;
@@ -2300 +3307 @@
                                                                         else if (*s == 0 && endch == '>')
                                                                         {
-                                                                                POPNODE();
+                                                                                PUGI__POPNODE();
                                                                                 break;
                                                                         }
-                                                                        else THROW_ERROR(status_bad_start_element, s);
+                                                                        else PUGI__THROW_ERROR(status_bad_start_element, s);
                                                                 }
                                                                 else if (*s == '>')
@@ -2315 +3322 @@
                                                                         break;
                                                                 }
-                                                                else THROW_ERROR(status_bad_start_element, s);
+                                                                else PUGI__THROW_ERROR(status_bad_start_element, s);
                                                         }
 
@@ -2322 +3329 @@
                                                 else if (ch == '/') // '<#.../'
                                                 {
-                                                        if (!ENDSWITH(*s, '>')) THROW_ERROR(status_bad_start_element, s);
-
-                                                        POPNODE(); // Pop.
+                                                        if (!PUGI__ENDSWITH(*s, '>')) PUGI__THROW_ERROR(status_bad_start_element, s);
+
+                                                        PUGI__POPNODE(); // Pop.
 
                                                         s += (*s == '>');
@@ -2332 +3339 @@
                                                         // we stepped over null terminator, backtrack & handle closing tag
                                                         --s;
-                                                        
-                                                        if (endch != '>') THROW_ERROR(status_bad_start_element, s);
+
+                                                        if (endch != '>') PUGI__THROW_ERROR(status_bad_start_element, s);
                                                 }
-                                                else THROW_ERROR(status_bad_start_element, s);
+                                                else PUGI__THROW_ERROR(status_bad_start_element, s);
                                         }
                                         else if (*s == '/')
@@ -2341 +3348 @@
                                                 ++s;
 
+                                                mark = s;
+
                                                 char_t* name = cursor->name;
-                                                if (!name) THROW_ERROR(status_end_element_mismatch, s);
-                                                
-                                                while (IS_CHARTYPE(*s, ct_symbol))
+                                                if (!name) PUGI__THROW_ERROR(status_end_element_mismatch, mark);
+
+                                                while (PUGI__IS_CHARTYPE(*s, ct_symbol))
                                                 {
-                                                        if (*s++ != *name++) THROW_ERROR(status_end_element_mismatch, s);
+                                                        if (*s++ != *name++) PUGI__THROW_ERROR(status_end_element_mismatch, mark);
                                                 }
 
                                                 if (*name)
                                                 {
-                                                        if (*s == 0 && name[0] == endch && name[1] == 0) THROW_ERROR(status_bad_end_element, s);
-                                                        else THROW_ERROR(status_end_element_mismatch, s);
+                                                        if (*s == 0 && name[0] == endch && name[1] == 0) PUGI__THROW_ERROR(status_bad_end_element, s);
+                                                        else PUGI__THROW_ERROR(status_end_element_mismatch, mark);
                                                 }
-                                                        
-                                                POPNODE(); // Pop.
-
-                                                SKIPWS();
+
+                                                PUGI__POPNODE(); // Pop.
+
+                                                PUGI__SKIPWS();
 
                                                 if (*s == 0)
                                                 {
-                                                        if (endch != '>') THROW_ERROR(status_bad_end_element, s);
+                                                        if (endch != '>') PUGI__THROW_ERROR(status_bad_end_element, s);
                                                 }
                                                 else
                                                 {
-                                                        if (*s != '>') THROW_ERROR(status_bad_end_element, s);
+                                                        if (*s != '>') PUGI__THROW_ERROR(status_bad_end_element, s);
                                                         ++s;
                                                 }
@@ -2372 +3381 @@
                                         {
                                                 s = parse_question(s, cursor, optmsk, endch);
+                                                if (!s) return s;
 
                                                 assert(cursor);
-                                                if ((cursor->header & xml_memory_page_type_mask) + 1 == node_declaration) goto LOC_ATTRIBUTES;
+                                                if (PUGI__NODETYPE(cursor) == node_declaration) goto LOC_ATTRIBUTES;
                                         }
                                         else if (*s == '!') // '<!...'
                                         {
                                                 s = parse_exclamation(s, cursor, optmsk, endch);
+                                                if (!s) return s;
                                         }
-                                        else if (*s == 0 && endch == '?') THROW_ERROR(status_bad_pi, s);
-                                        else THROW_ERROR(status_unrecognized_tag, s);
+                                        else if (*s == 0 && endch == '?') PUGI__THROW_ERROR(status_bad_pi, s);
+                                        else PUGI__THROW_ERROR(status_unrecognized_tag, s);
                                 }
                                 else
@@ -2387 +3398 @@
                                         mark = s; // Save this offset while searching for a terminator.
 
-                                        SKIPWS(); // Eat whitespace if no genuine PCDATA here.
-
-                                        if ((!OPTSET(parse_ws_pcdata) || mark == s) && (*s == '<' || !*s))
+                                        PUGI__SKIPWS(); // Eat whitespace if no genuine PCDATA here.
+
+                                        if (*s == '<' || !*s)
                                         {
-                                                continue;
+                                                // We skipped some whitespace characters because otherwise we would take the tag branch instead of PCDATA one
+                                                assert(mark != s);
+
+                                                if (!PUGI__OPTSET(parse_ws_pcdata | parse_ws_pcdata_single) || PUGI__OPTSET(parse_trim_pcdata))
+                                                {
+                                                        continue;
+                                                }
+                                                else if (PUGI__OPTSET(parse_ws_pcdata_single))
+                                                {
+                                                        if (s[0] != '<' || s[1] != '/' || cursor->first_child) continue;
+                                                }
                                         }
 
-                                        s = mark;
-                                                        
-                                        if (cursor->parent)
+                                        if (!PUGI__OPTSET(parse_trim_pcdata))
+                                                s = mark;
+
+                                        if (cursor->parent || PUGI__OPTSET(parse_fragment))
                                         {
-                                                PUSHNODE(node_pcdata); // Append a new node on the tree.
-                                                cursor->value = s; // Save the offset.
+                                                if (PUGI__OPTSET(parse_embed_pcdata) && cursor->parent && !cursor->first_child && !cursor->value)
+                                                {
+                                                        cursor->value = s; // Save the offset.
+                                                }
+                                                else
+                                                {
+                                                        PUGI__PUSHNODE(node_pcdata); // Append a new node on the tree.
+
+                                                        cursor->value = s; // Save the offset.
+
+                                                        PUGI__POPNODE(); // Pop since this is a standalone.
+                                                }
 
                                                 s = strconv_pcdata(s);
-                                                                
-                                                POPNODE(); // Pop since this is a standalone.
-                                                
+
                                                 if (!*s) break;
                                         }
                                         else
                                         {
-                                                SCANFOR(*s == '<'); // '...<'
+                                                PUGI__SCANFOR(*s == '<'); // '...<'
                                                 if (!*s) break;
-                                                
+
                                                 ++s;
                                         }
@@ -2421 +3451 @@
 
                         // check that last tag is closed
-                        if (cursor != xmldoc) THROW_ERROR(status_end_element_mismatch, s);
-                }
-
-                static xml_parse_result parse(char_t* buffer, size_t length, xml_node_struct* root, unsigned int optmsk)
-                {
-                        xml_document_struct* xmldoc = static_cast<xml_document_struct*>(root);
-
-                        // store buffer for offset_debug
-                        xmldoc->buffer = buffer;
-
+                        if (cursor != root) PUGI__THROW_ERROR(status_end_element_mismatch, s);
+
+                        return s;
+                }
+
+        #ifdef PUGIXML_WCHAR_MODE
+                static char_t* parse_skip_bom(char_t* s)
+                {
+                        unsigned int bom = 0xfeff;
+                        return (s[0] == static_cast<wchar_t>(bom)) ? s + 1 : s;
+                }
+        #else
+                static char_t* parse_skip_bom(char_t* s)
+                {
+                        return (s[0] == '\xef' && s[1] == '\xbb' && s[2] == '\xbf') ? s + 3 : s;
+                }
+        #endif
+
+                static bool has_element_node_siblings(xml_node_struct* node)
+                {
+                        while (node)
+                        {
+                                if (PUGI__NODETYPE(node) == node_element) return true;
+
+                                node = node->next_sibling;
+                        }
+
+                        return false;
+                }
+
+                static xml_parse_result parse(char_t* buffer, size_t length, xml_document_struct* xmldoc, xml_node_struct* root, unsigned int optmsk)
+                {
                         // early-out for empty documents
-                        if (length == 0) return make_parse_result(status_ok);
+                        if (length == 0)
+                                return make_parse_result(PUGI__OPTSET(parse_fragment) ? status_ok : status_no_document_element);
+
+                        // get last child of the root before parsing
+                        xml_node_struct* last_root_child = root->first_child ? root->first_child->prev_sibling_c + 0 : 0;
 
                         // create parser on stack
-                        xml_parser parser(*xmldoc);
+                        xml_parser parser(static_cast<xml_allocator*>(xmldoc));
 
                         // save last character and make buffer zero-terminated (speeds up parsing)
                         char_t endch = buffer[length - 1];
                         buffer[length - 1] = 0;
-                        
+
+                        // skip BOM to make sure it does not end up as part of parse output
+                        char_t* buffer_data = parse_skip_bom(buffer);
+
                         // perform actual parsing
-                        int error = setjmp(parser.error_handler);
-
-                        if (error == 0)
-                        {
-                                parser.parse(buffer, xmldoc, optmsk, endch);
-                        }
-
-                        xml_parse_result result = make_parse_result(static_cast<xml_parse_status>(error), parser.error_offset ? parser.error_offset - buffer : 0);
+                        parser.parse_tree(buffer_data, root, optmsk, endch);
+
+                        xml_parse_result result = make_parse_result(parser.error_status, parser.error_offset ? parser.error_offset - buffer : 0);
                         assert(result.offset >= 0 && static_cast<size_t>(result.offset) <= length);
 
-                        // update allocator state
-                        *static_cast<xml_allocator*>(xmldoc) = parser.alloc;
-
-                        // since we removed last character, we have to handle the only possible false positive
-                        if (result && endch == '<')
-                        {
-                                // there's no possible well-formed document with < at the end
-                                return make_parse_result(status_unrecognized_tag, length);
+                        if (result)
+                        {
+                                // since we removed last character, we have to handle the only possible false positive (stray <)
+                                if (endch == '<')
+                                        return make_parse_result(status_unrecognized_tag, length - 1);
+
+                                // check if there are any element nodes parsed
+                                xml_node_struct* first_root_child_parsed = last_root_child ? last_root_child->next_sibling + 0 : root->first_child+ 0;
+
+                                if (!PUGI__OPTSET(parse_fragment) && !has_element_node_siblings(first_root_child_parsed))
+                                        return make_parse_result(status_no_document_element, length - 1);
+                        }
+                        else
+                        {
+                                // roll back offset if it occurs on a null terminator in the source buffer
+                                if (result.offset > 0 && static_cast<size_t>(result.offset) == length - 1 && endch == 0)
+                                        result.offset--;
                         }
 
@@ -2467 +3530 @@
 
         // Output facilities
-        xml_encoding get_write_native_encoding()
+        PUGI__FN xml_encoding get_write_native_encoding()
         {
         #ifdef PUGIXML_WCHAR_MODE
@@ -2476 +3539 @@
         }
 
-        xml_encoding get_write_encoding(xml_encoding encoding)
+        PUGI__FN xml_encoding get_write_encoding(xml_encoding encoding)
         {
                 // replace wchar encoding with utf implementation
@@ -2494 +3557 @@
         }
 
+        template <typename D, typename T> PUGI__FN size_t convert_buffer_output_generic(typename T::value_type dest, const char_t* data, size_t length, D, T)
+        {
+                PUGI__STATIC_ASSERT(sizeof(char_t) == sizeof(typename D::type));
+
+                typename T::value_type end = D::process(reinterpret_cast<const typename D::type*>(data), length, dest, T());
+
+                return static_cast<size_t>(end - dest) * sizeof(*dest);
+        }
+
+        template <typename D, typename T> PUGI__FN size_t convert_buffer_output_generic(typename T::value_type dest, const char_t* data, size_t length, D, T, bool opt_swap)
+        {
+                PUGI__STATIC_ASSERT(sizeof(char_t) == sizeof(typename D::type));
+
+                typename T::value_type end = D::process(reinterpret_cast<const typename D::type*>(data), length, dest, T());
+
+                if (opt_swap)
+                {
+                        for (typename T::value_type i = dest; i != end; ++i)
+                                *i = endian_swap(*i);
+                }
+
+                return static_cast<size_t>(end - dest) * sizeof(*dest);
+        }
+
 #ifdef PUGIXML_WCHAR_MODE
-        size_t get_valid_length(const char_t* data, size_t length)
-        {
-                assert(length > 0);
-
-                // discard last character if it's the lead of a surrogate pair 
-                return (sizeof(wchar_t) == 2 && (unsigned)(static_cast<uint16_t>(data[length - 1]) - 0xD800) < 0x400) ? length - 1 : length;
-        }
-
-        size_t convert_buffer(char* result, const char_t* data, size_t length, xml_encoding encoding)
+        PUGI__FN size_t get_valid_length(const char_t* data, size_t length)
+        {
+                if (length < 1) return 0;
+
+                // discard last character if it's the lead of a surrogate pair
+                return (sizeof(wchar_t) == 2 && static_cast<unsigned int>(static_cast<uint16_t>(data[length - 1]) - 0xD800) < 0x400) ? length - 1 : length;
+        }
+
+        PUGI__FN size_t convert_buffer_output(char_t* r_char, uint8_t* r_u8, uint16_t* r_u16, uint32_t* r_u32, const char_t* data, size_t length, xml_encoding encoding)
         {
                 // only endian-swapping is required
                 if (need_endian_swap_utf(encoding, get_wchar_encoding()))
                 {
-                        convert_wchar_endian_swap(reinterpret_cast<char_t*>(result), data, length);
+                        convert_wchar_endian_swap(r_char, data, length);
 
                         return length * sizeof(char_t);
                 }
-        
+
                 // convert to utf8
                 if (encoding == encoding_utf8)
-                {
-                        uint8_t* dest = reinterpret_cast<uint8_t*>(result);
-
-                        uint8_t* end = sizeof(wchar_t) == 2 ?
-                                utf_decoder<utf8_writer>::decode_utf16_block(reinterpret_cast<const uint16_t*>(data), length, dest) :
-                                utf_decoder<utf8_writer>::decode_utf32_block(reinterpret_cast<const uint32_t*>(data), length, dest);
-
-                        return static_cast<size_t>(end - dest);
-                }
+                        return convert_buffer_output_generic(r_u8, data, length, wchar_decoder(), utf8_writer());
 
                 // convert to utf16
                 if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
                 {
-                        uint16_t* dest = reinterpret_cast<uint16_t*>(result);
-
-                        // convert to native utf16
-                        uint16_t* end = utf_decoder<utf16_writer>::decode_utf32_block(reinterpret_cast<const uint32_t*>(data), length, dest);
-
-                        // swap if necessary
                         xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
 
-                        if (native_encoding != encoding) convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
-
-                        return static_cast<size_t>(end - dest) * sizeof(uint16_t);
+                        return convert_buffer_output_generic(r_u16, data, length, wchar_decoder(), utf16_writer(), native_encoding != encoding);
                 }
 
@@ -2544 +3615 @@
                 if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
                 {
-                        uint32_t* dest = reinterpret_cast<uint32_t*>(result);
-
-                        // convert to native utf32
-                        uint32_t* end = utf_decoder<utf32_writer>::decode_utf16_block(reinterpret_cast<const uint16_t*>(data), length, dest);
-
-                        // swap if necessary
                         xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
 
-                        if (native_encoding != encoding) convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
-
-                        return static_cast<size_t>(end - dest) * sizeof(uint32_t);
-                }
-
-                assert(!"Invalid encoding");
+                        return convert_buffer_output_generic(r_u32, data, length, wchar_decoder(), utf32_writer(), native_encoding != encoding);
+                }
+
+                // convert to latin1
+                if (encoding == encoding_latin1)
+                        return convert_buffer_output_generic(r_u8, data, length, wchar_decoder(), latin1_writer());
+
+                assert(false && "Invalid encoding");
                 return 0;
         }
 #else
-        size_t get_valid_length(const char_t* data, size_t length)
-        {
-                assert(length > 4);
+        PUGI__FN size_t get_valid_length(const char_t* data, size_t length)
+        {
+                if (length < 5) return 0;
 
                 for (size_t i = 1; i <= 4; ++i)
@@ -2577 +3644 @@
         }
 
-        size_t convert_buffer(char* result, const char_t* data, size_t length, xml_encoding encoding)
+        PUGI__FN size_t convert_buffer_output(char_t* /* r_char */, uint8_t* r_u8, uint16_t* r_u16, uint32_t* r_u32, const char_t* data, size_t length, xml_encoding encoding)
         {
                 if (encoding == encoding_utf16_be || encoding == encoding_utf16_le)
                 {
-                        uint16_t* dest = reinterpret_cast<uint16_t*>(result);
-
-                        // convert to native utf16
-                        uint16_t* end = utf_decoder<utf16_writer>::decode_utf8_block(reinterpret_cast<const uint8_t*>(data), length, dest);
-
-                        // swap if necessary
                         xml_encoding native_encoding = is_little_endian() ? encoding_utf16_le : encoding_utf16_be;
 
-                        if (native_encoding != encoding) convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
-
-                        return static_cast<size_t>(end - dest) * sizeof(uint16_t);
+                        return convert_buffer_output_generic(r_u16, data, length, utf8_decoder(), utf16_writer(), native_encoding != encoding);
                 }
 
                 if (encoding == encoding_utf32_be || encoding == encoding_utf32_le)
                 {
-                        uint32_t* dest = reinterpret_cast<uint32_t*>(result);
-
-                        // convert to native utf32
-                        uint32_t* end = utf_decoder<utf32_writer>::decode_utf8_block(reinterpret_cast<const uint8_t*>(data), length, dest);
-
-                        // swap if necessary
                         xml_encoding native_encoding = is_little_endian() ? encoding_utf32_le : encoding_utf32_be;
 
-                        if (native_encoding != encoding) convert_utf_endian_swap(dest, dest, static_cast<size_t>(end - dest));
-
-                        return static_cast<size_t>(end - dest) * sizeof(uint32_t);
-                }
-
-                assert(!"Invalid encoding");
+                        return convert_buffer_output_generic(r_u32, data, length, utf8_decoder(), utf32_writer(), native_encoding != encoding);
+                }
+
+                if (encoding == encoding_latin1)
+                        return convert_buffer_output_generic(r_u8, data, length, utf8_decoder(), latin1_writer());
+
+                assert(false && "Invalid encoding");
                 return 0;
         }
@@ -2620 +3674 @@
 
         public:
-                xml_buffered_writer(xml_writer& writer, xml_encoding user_encoding): writer(writer), bufsize(0), encoding(get_write_encoding(user_encoding))
-                {
-                }
-
-                ~xml_buffered_writer()
-                {
-                        flush();
-                }
-
-                void flush()
+                xml_buffered_writer(xml_writer& writer_, xml_encoding user_encoding): writer(writer_), bufsize(0), encoding(get_write_encoding(user_encoding))
+                {
+                        PUGI__STATIC_ASSERT(bufcapacity >= 8);
+                }
+
+                size_t flush()
                 {
                         flush(buffer, bufsize);
                         bufsize = 0;
+                        return 0;
                 }
 
@@ -2645 +3696 @@
                         {
                                 // convert chunk
-                                size_t result = convert_buffer(scratch, data, size, encoding);
+                                size_t result = convert_buffer_output(scratch.data_char, scratch.data_u8, scratch.data_u16, scratch.data_u32, data, size, encoding);
                                 assert(result <= sizeof(scratch));
 
                                 // write data
-                                writer.write(scratch, result);
-                        }
-                }
-
-                void write(const char_t* data, size_t length)
-                {
-                        if (bufsize + length > bufcapacity)
-                        {
-                                // flush the remaining buffer contents
-                                flush();
-
-                                // handle large chunks
-                                if (length > bufcapacity)
-                                {
-                                        if (encoding == get_write_native_encoding())
-                                        {
-                                                // fast path, can just write data chunk
-                                                writer.write(data, length * sizeof(char_t));
-                                                return;
-                                        }
-
-                                        // need to convert in suitable chunks
-                                        while (length > bufcapacity)
-                                        {
-                                                // get chunk size by selecting such number of characters that are guaranteed to fit into scratch buffer
-                                                // and form a complete codepoint sequence (i.e. discard start of last codepoint if necessary)
-                                                size_t chunk_size = get_valid_length(data, bufcapacity);
-
-                                                // convert chunk and write
-                                                flush(data, chunk_size);
-
-                                                // iterate
-                                                data += chunk_size;
-                                                length -= chunk_size;
-                                        }
-
-                                        // small tail is copied below
-                                        bufsize = 0;
-                                }
+                                writer.write(scratch.data_u8, result);
+                        }
+                }
+
+                void write_direct(const char_t* data, size_t length)
+                {
+                        // flush the remaining buffer contents
+                        flush();
+
+                        // handle large chunks
+                        if (length > bufcapacity)
+                        {
+                                if (encoding == get_write_native_encoding())
+                                {
+                                        // fast path, can just write data chunk
+                                        writer.write(data, length * sizeof(char_t));
+                                        return;
+                                }
+
+                                // need to convert in suitable chunks
+                                while (length > bufcapacity)
+                                {
+                                        // get chunk size by selecting such number of characters that are guaranteed to fit into scratch buffer
+                                        // and form a complete codepoint sequence (i.e. discard start of last codepoint if necessary)
+                                        size_t chunk_size = get_valid_length(data, bufcapacity);
+                                        assert(chunk_size);
+
+                                        // convert chunk and write
+                                        flush(data, chunk_size);
+
+                                        // iterate
+                                        data += chunk_size;
+                                        length -= chunk_size;
+                                }
+
+                                // small tail is copied below
+                                bufsize = 0;
                         }
 
@@ -2694 +3743 @@
                 }
 
-                void write(const char_t* data)
-                {
-                        write(data, strlength(data));
+                void write_buffer(const char_t* data, size_t length)
+                {
+                        size_t offset = bufsize;
+
+                        if (offset + length <= bufcapacity)
+                        {
+                                memcpy(buffer + offset, data, length * sizeof(char_t));
+                                bufsize = offset + length;
+                        }
+                        else
+                        {
+                                write_direct(data, length);
+                        }
+                }
+
+                void write_string(const char_t* data)
+                {
+                        // write the part of the string that fits in the buffer
+                        size_t offset = bufsize;
+
+                        while (*data && offset < bufcapacity)
+                                buffer[offset++] = *data++;
+
+                        // write the rest
+                        if (offset < bufcapacity)
+                        {
+                                bufsize = offset;
+                        }
+                        else
+                        {
+                                // backtrack a bit if we have split the codepoint
+                                size_t length = offset - bufsize;
+                                size_t extra = length - get_valid_length(data - length, length);
+
+                                bufsize = offset - extra;
+
+                                write_direct(data - extra, strlength(data) + extra);
+                        }
                 }
 
                 void write(char_t d0)
                 {
-                        if (bufsize + 1 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        bufsize += 1;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 1) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        bufsize = offset + 1;
                 }
 
                 void write(char_t d0, char_t d1)
                 {
-                        if (bufsize + 2 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        buffer[bufsize + 1] = d1;
-                        bufsize += 2;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 2) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        buffer[offset + 1] = d1;
+                        bufsize = offset + 2;
                 }
 
                 void write(char_t d0, char_t d1, char_t d2)
                 {
-                        if (bufsize + 3 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        buffer[bufsize + 1] = d1;
-                        buffer[bufsize + 2] = d2;
-                        bufsize += 3;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 3) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        buffer[offset + 1] = d1;
+                        buffer[offset + 2] = d2;
+                        bufsize = offset + 3;
                 }
 
                 void write(char_t d0, char_t d1, char_t d2, char_t d3)
                 {
-                        if (bufsize + 4 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        buffer[bufsize + 1] = d1;
-                        buffer[bufsize + 2] = d2;
-                        buffer[bufsize + 3] = d3;
-                        bufsize += 4;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 4) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        buffer[offset + 1] = d1;
+                        buffer[offset + 2] = d2;
+                        buffer[offset + 3] = d3;
+                        bufsize = offset + 4;
                 }
 
                 void write(char_t d0, char_t d1, char_t d2, char_t d3, char_t d4)
                 {
-                        if (bufsize + 5 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        buffer[bufsize + 1] = d1;
-                        buffer[bufsize + 2] = d2;
-                        buffer[bufsize + 3] = d3;
-                        buffer[bufsize + 4] = d4;
-                        bufsize += 5;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 5) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        buffer[offset + 1] = d1;
+                        buffer[offset + 2] = d2;
+                        buffer[offset + 3] = d3;
+                        buffer[offset + 4] = d4;
+                        bufsize = offset + 5;
                 }
 
                 void write(char_t d0, char_t d1, char_t d2, char_t d3, char_t d4, char_t d5)
                 {
-                        if (bufsize + 6 > bufcapacity) flush();
-
-                        buffer[bufsize + 0] = d0;
-                        buffer[bufsize + 1] = d1;
-                        buffer[bufsize + 2] = d2;
-                        buffer[bufsize + 3] = d3;
-                        buffer[bufsize + 4] = d4;
-                        buffer[bufsize + 5] = d5;
-                        bufsize += 6;
+                        size_t offset = bufsize;
+                        if (offset > bufcapacity - 6) offset = flush();
+
+                        buffer[offset + 0] = d0;
+                        buffer[offset + 1] = d1;
+                        buffer[offset + 2] = d2;
+                        buffer[offset + 3] = d3;
+                        buffer[offset + 4] = d4;
+                        buffer[offset + 5] = d5;
+                        bufsize = offset + 6;
                 }
 
@@ -2765 +3855 @@
                 // utf16 maximum expansion: x2 (-> utf32)
                 // utf32 maximum expansion: x1
-                enum { bufcapacity = 2048 };
+                enum
+                {
+                        bufcapacitybytes =
+                        #ifdef PUGIXML_MEMORY_OUTPUT_STACK
+                                PUGIXML_MEMORY_OUTPUT_STACK
+                        #else
+                                10240
+                        #endif
+                        ,
+                        bufcapacity = bufcapacitybytes / (sizeof(char_t) + 4)
+                };
 
                 char_t buffer[bufcapacity];
-                char scratch[4 * bufcapacity];
+
+                union
+                {
+                        uint8_t data_u8[4 * bufcapacity];
+                        uint16_t data_u16[2 * bufcapacity];
+                        uint32_t data_u32[bufcapacity];
+                        char_t data_char[bufcapacity];
+                } scratch;
 
                 xml_writer& writer;
@@ -2775 +3882 @@
         };
 
-        void write_bom(xml_writer& writer, xml_encoding encoding)
-        {
-                switch (encoding)
-                {
-                case encoding_utf8:
-                        writer.write("\xef\xbb\xbf", 3);
-                        break;
-
-                case encoding_utf16_be:
-                        writer.write("\xfe\xff", 2);
-                        break;
-
-                case encoding_utf16_le:
-                        writer.write("\xff\xfe", 2);
-                        break;
-
-                case encoding_utf32_be:
-                        writer.write("\x00\x00\xfe\xff", 4);
-                        break;
-
-                case encoding_utf32_le:
-                        writer.write("\xff\xfe\x00\x00", 4);
-                        break;
-
-                default:
-                        assert(!"Invalid encoding");
-                }
-        }
-
-        void text_output_escaped(xml_buffered_writer& writer, const char_t* s, chartypex_t type)
+        PUGI__FN void text_output_escaped(xml_buffered_writer& writer, const char_t* s, chartypex_t type)
         {
                 while (*s)
                 {
                         const char_t* prev = s;
-                        
+
                         // While *s is a usual symbol
-                        while (!IS_CHARTYPEX(*s, type)) ++s;
-                
-                        writer.write(prev, static_cast<size_t>(s - prev));
+                        PUGI__SCANWHILE_UNROLL(!PUGI__IS_CHARTYPEX(ss, type));
+
+                        writer.write_buffer(prev, static_cast<size_t>(s - prev));
 
                         switch (*s)
@@ -2845 +3923 @@
         }
 
-        void text_output_cdata(xml_buffered_writer& writer, const char_t* s)
+        PUGI__FN void text_output(xml_buffered_writer& writer, const char_t* s, chartypex_t type, unsigned int flags)
+        {
+                if (flags & format_no_escapes)
+                        writer.write_string(s);
+                else
+                        text_output_escaped(writer, s, type);
+        }
+
+        PUGI__FN void text_output_cdata(xml_buffered_writer& writer, const char_t* s)
         {
                 do
@@ -2860 +3946 @@
                         if (*s) s += 2;
 
-                        writer.write(prev, static_cast<size_t>(s - prev));
+                        writer.write_buffer(prev, static_cast<size_t>(s - prev));
 
                         writer.write(']', ']', '>');
@@ -2867 +3953 @@
         }
 
-        void node_output_attributes(xml_buffered_writer& writer, const xml_node& node)
+        PUGI__FN void text_output_indent(xml_buffered_writer& writer, const char_t* indent, size_t indent_length, unsigned int depth)
+        {
+                switch (indent_length)
+                {
+                case 1:
+                {
+                        for (unsigned int i = 0; i < depth; ++i)
+                                writer.write(indent[0]);
+                        break;
+                }
+
+                case 2:
+                {
+                        for (unsigned int i = 0; i < depth; ++i)
+                                writer.write(indent[0], indent[1]);
+                        break;
+                }
+
+                case 3:
+                {
+                        for (unsigned int i = 0; i < depth; ++i)
+                                writer.write(indent[0], indent[1], indent[2]);
+                        break;
+                }
+
+                case 4:
+                {
+                        for (unsigned int i = 0; i < depth; ++i)
+                                writer.write(indent[0], indent[1], indent[2], indent[3]);
+                        break;
+                }
+
+                default:
+                {
+                        for (unsigned int i = 0; i < depth; ++i)
+                                writer.write_buffer(indent, indent_length);
+                }
+                }
+        }
+
+        PUGI__FN void node_output_comment(xml_buffered_writer& writer, const char_t* s)
+        {
+                writer.write('<', '!', '-', '-');
+
+                while (*s)
+                {
+                        const char_t* prev = s;
+
+                        // look for -\0 or -- sequence - we can't output it since -- is illegal in comment body
+                        while (*s && !(s[0] == '-' && (s[1] == '-' || s[1] == 0))) ++s;
+
+                        writer.write_buffer(prev, static_cast<size_t>(s - prev));
+
+                        if (*s)
+                        {
+                                assert(*s == '-');
+
+                                writer.write('-', ' ');
+                                ++s;
+                        }
+                }
+
+                writer.write('-', '-', '>');
+        }
+
+        PUGI__FN void node_output_pi_value(xml_buffered_writer& writer, const char_t* s)
+        {
+                while (*s)
+                {
+                        const char_t* prev = s;
+
+                        // look for ?> sequence - we can't output it since ?> terminates PI
+                        while (*s && !(s[0] == '?' && s[1] == '>')) ++s;
+
+                        writer.write_buffer(prev, static_cast<size_t>(s - prev));
+
+                        if (*s)
+                        {
+                                assert(s[0] == '?' && s[1] == '>');
+
+                                writer.write('?', ' ', '>');
+                                s += 2;
+                        }
+                }
+        }
+
+        PUGI__FN void node_output_attributes(xml_buffered_writer& writer, xml_node_struct* node, const char_t* indent, size_t indent_length, unsigned int flags, unsigned int depth)
         {
                 const char_t* default_name = PUGIXML_TEXT(":anonymous");
 
-                for (xml_attribute a = node.first_attribute(); a; a = a.next_attribute())
-                {
-                        writer.write(' ');
-                        writer.write(a.name()[0] ? a.name() : default_name);
+                for (xml_attribute_struct* a = node->first_attribute; a; a = a->next_attribute)
+                {
+                        if ((flags & (format_indent_attributes | format_raw)) == format_indent_attributes)
+                        {
+                                writer.write('\n');
+
+                                text_output_indent(writer, indent, indent_length, depth + 1);
+                        }
+                        else
+                        {
+                                writer.write(' ');
+                        }
+
+                        writer.write_string(a->name ? a->name + 0 : default_name);
                         writer.write('=', '"');
 
-                        text_output_escaped(writer, a.value(), ctx_special_attr);
+                        if (a->value)
+                                text_output(writer, a->value, ctx_special_attr, flags);
 
                         writer.write('"');
@@ -2883 +4066 @@
         }
 
-        void node_output(xml_buffered_writer& writer, const xml_node& node, const char_t* indent, unsigned int flags, unsigned int depth)
+        PUGI__FN bool node_output_start(xml_buffered_writer& writer, xml_node_struct* node, const char_t* indent, size_t indent_length, unsigned int flags, unsigned int depth)
         {
                 const char_t* default_name = PUGIXML_TEXT(":anonymous");
-
-                if ((flags & format_indent) != 0 && (flags & format_raw) == 0)
-                        for (unsigned int i = 0; i < depth; ++i) writer.write(indent);
-
-                switch (node.type())
-                {
-                case node_document:
-                {
-                        for (xml_node n = node.first_child(); n; n = n.next_sibling())
-                                node_output(writer, n, indent, flags, depth);
-                        break;
-                }
-                        
-                case node_element:
-                {
-                        const char_t* name = node.name()[0] ? node.name() : default_name;
-
-                        writer.write('<');
-                        writer.write(name);
-
-                        node_output_attributes(writer, node);
-
-                        if (flags & format_raw)
-                        {
-                                if (!node.first_child())
-                                        writer.write(' ', '/', '>');
+                const char_t* name = node->name ? node->name + 0 : default_name;
+
+                writer.write('<');
+                writer.write_string(name);
+
+                if (node->first_attribute)
+                        node_output_attributes(writer, node, indent, indent_length, flags, depth);
+
+                // element nodes can have value if parse_embed_pcdata was used
+                if (!node->value)
+                {
+                        if (!node->first_child)
+                        {
+                                if (flags & format_no_empty_element_tags)
+                                {
+                                        writer.write('>', '<', '/');
+                                        writer.write_string(name);
+                                        writer.write('>');
+
+                                        return false;
+                                }
                                 else
                                 {
-                                        writer.write('>');
-
-                                        for (xml_node n = node.first_child(); n; n = n.next_sibling())
-                                                node_output(writer, n, indent, flags, depth + 1);
-
-                                        writer.write('<', '/');
-                                        writer.write(name);
-                                        writer.write('>');
-                                }
-                        }
-                        else if (!node.first_child())
-                                writer.write(' ', '/', '>', '\n');
-                        else if (node.first_child() == node.last_child() && (node.first_child().type() == node_pcdata || node.first_child().type() == node_cdata))
+                                        if ((flags & format_raw) == 0)
+                                                writer.write(' ');
+
+                                        writer.write('/', '>');
+
+                                        return false;
+                                }
+                        }
+                        else
                         {
                                 writer.write('>');
 
-                if (node.first_child().type() == node_pcdata)
-                    text_output_escaped(writer, node.first_child().value(), ctx_special_pcdata);
-                else
-                    text_output_cdata(writer, node.first_child().value());
-
+                                return true;
+                        }
+                }
+                else
+                {
+                        writer.write('>');
+
+                        text_output(writer, node->value, ctx_special_pcdata, flags);
+
+                        if (!node->first_child)
+                        {
                                 writer.write('<', '/');
-                                writer.write(name);
-                                writer.write('>', '\n');
+                                writer.write_string(name);
+                                writer.write('>');
+
+                                return false;
                         }
                         else
                         {
-                                writer.write('>', '\n');
-                                
-                                for (xml_node n = node.first_child(); n; n = n.next_sibling())
-                                        node_output(writer, n, indent, flags, depth + 1);
-
-                                if ((flags & format_indent) != 0 && (flags & format_raw) == 0)
-                                        for (unsigned int i = 0; i < depth; ++i) writer.write(indent);
-                                
-                                writer.write('<', '/');
-                                writer.write(name);
-                                writer.write('>', '\n');
-                        }
-
-                        break;
-                }
-                
-                case node_pcdata:
-                        text_output_escaped(writer, node.value(), ctx_special_pcdata);
-                        if ((flags & format_raw) == 0) writer.write('\n');
-                        break;
-
-                case node_cdata:
-                        text_output_cdata(writer, node.value());
-                        if ((flags & format_raw) == 0) writer.write('\n');
-                        break;
-
-                case node_comment:
-                        writer.write('<', '!', '-', '-');
-                        writer.write(node.value());
-                        writer.write('-', '-', '>');
-                        if ((flags & format_raw) == 0) writer.write('\n');
-                        break;
-
-                case node_pi:
-                case node_declaration:
-                        writer.write('<', '?');
-                        writer.write(node.name()[0] ? node.name() : default_name);
-
-                        if (node.type() == node_declaration)
-                        {
-                                node_output_attributes(writer, node);
-                        }
-                        else if (node.value()[0])
-                        {
-                                writer.write(' ');
-                                writer.write(node.value());
-                        }
-
-                        writer.write('?', '>');
-                        if ((flags & format_raw) == 0) writer.write('\n');
-                        break;
-
-                case node_doctype:
-                        writer.write('<', '!', 'D', 'O', 'C');
-                        writer.write('T', 'Y', 'P', 'E');
-
-            if (node.value()[0])
-            {
-                writer.write(' ');
-                writer.write(node.value());
-            }
-
-            writer.write('>');
-                        if ((flags & format_raw) == 0) writer.write('\n');
-                        break;
-
-                default:
-                        assert(!"Invalid node type");
-                }
-        }
-
-        inline bool has_declaration(const xml_node& node)
-        {
-                for (xml_node child = node.first_child(); child; child = child.next_sibling())
-                {
-                        xml_node_type type = child.type();
+                                return true;
+                        }
+                }
+        }
+
+        PUGI__FN void node_output_end(xml_buffered_writer& writer, xml_node_struct* node)
+        {
+                const char_t* default_name = PUGIXML_TEXT(":anonymous");
+                const char_t* name = node->name ? node->name + 0 : default_name;
+
+                writer.write('<', '/');
+                writer.write_string(name);
+                writer.write('>');
+        }
+
+        PUGI__FN void node_output_simple(xml_buffered_writer& writer, xml_node_struct* node, unsigned int flags)
+        {
+                const char_t* default_name = PUGIXML_TEXT(":anonymous");
+
+                switch (PUGI__NODETYPE(node))
+                {
+                        case node_pcdata:
+                                text_output(writer, node->value ? node->value + 0 : PUGIXML_TEXT(""), ctx_special_pcdata, flags);
+                                break;
+
+                        case node_cdata:
+                                text_output_cdata(writer, node->value ? node->value + 0 : PUGIXML_TEXT(""));
+                                break;
+
+                        case node_comment:
+                                node_output_comment(writer, node->value ? node->value + 0 : PUGIXML_TEXT(""));
+                                break;
+
+                        case node_pi:
+                                writer.write('<', '?');
+                                writer.write_string(node->name ? node->name + 0 : default_name);
+
+                                if (node->value)
+                                {
+                                        writer.write(' ');
+                                        node_output_pi_value(writer, node->value);
+                                }
+
+                                writer.write('?', '>');
+                                break;
+
+                        case node_declaration:
+                                writer.write('<', '?');
+                                writer.write_string(node->name ? node->name + 0 : default_name);
+                                node_output_attributes(writer, node, PUGIXML_TEXT(""), 0, flags | format_raw, 0);
+                                writer.write('?', '>');
+                                break;
+
+                        case node_doctype:
+                                writer.write('<', '!', 'D', 'O', 'C');
+                                writer.write('T', 'Y', 'P', 'E');
+
+                                if (node->value)
+                                {
+                                        writer.write(' ');
+                                        writer.write_string(node->value);
+                                }
+
+                                writer.write('>');
+                                break;
+
+                        default:
+                                assert(false && "Invalid node type");
+                }
+        }
+
+        enum indent_flags_t
+        {
+                indent_newline = 1,
+                indent_indent = 2
+        };
+
+        PUGI__FN void node_output(xml_buffered_writer& writer, xml_node_struct* root, const char_t* indent, unsigned int flags, unsigned int depth)
+        {
+                size_t indent_length = ((flags & (format_indent | format_indent_attributes)) && (flags & format_raw) == 0) ? strlength(indent) : 0;
+                unsigned int indent_flags = indent_indent;
+
+                xml_node_struct* node = root;
+
+                do
+                {
+                        assert(node);
+
+                        // begin writing current node
+                        if (PUGI__NODETYPE(node) == node_pcdata || PUGI__NODETYPE(node) == node_cdata)
+                        {
+                                node_output_simple(writer, node, flags);
+
+                                indent_flags = 0;
+                        }
+                        else
+                        {
+                                if ((indent_flags & indent_newline) && (flags & format_raw) == 0)
+                                        writer.write('\n');
+
+                                if ((indent_flags & indent_indent) && indent_length)
+                                        text_output_indent(writer, indent, indent_length, depth);
+
+                                if (PUGI__NODETYPE(node) == node_element)
+                                {
+                                        indent_flags = indent_newline | indent_indent;
+
+                                        if (node_output_start(writer, node, indent, indent_length, flags, depth))
+                                        {
+                                                // element nodes can have value if parse_embed_pcdata was used
+                                                if (node->value)
+                                                        indent_flags = 0;
+
+                                                node = node->first_child;
+                                                depth++;
+                                                continue;
+                                        }
+                                }
+                                else if (PUGI__NODETYPE(node) == node_document)
+                                {
+                                        indent_flags = indent_indent;
+
+                                        if (node->first_child)
+                                        {
+                                                node = node->first_child;
+                                                continue;
+                                        }
+                                }
+                                else
+                                {
+                                        node_output_simple(writer, node, flags);
+
+                                        indent_flags = indent_newline | indent_indent;
+                                }
+                        }
+
+                        // continue to the next node
+                        while (node != root)
+                        {
+                                if (node->next_sibling)
+                                {
+                                        node = node->next_sibling;
+                                        break;
+                                }
+
+                                node = node->parent;
+
+                                // write closing node
+                                if (PUGI__NODETYPE(node) == node_element)
+                                {
+                                        depth--;
+
+                                        if ((indent_flags & indent_newline) && (flags & format_raw) == 0)
+                                                writer.write('\n');
+
+                                        if ((indent_flags & indent_indent) && indent_length)
+                                                text_output_indent(writer, indent, indent_length, depth);
+
+                                        node_output_end(writer, node);
+
+                                        indent_flags = indent_newline | indent_indent;
+                                }
+                        }
+                }
+                while (node != root);
+
+                if ((indent_flags & indent_newline) && (flags & format_raw) == 0)
+                        writer.write('\n');
+        }
+
+        PUGI__FN bool has_declaration(xml_node_struct* node)
+        {
+                for (xml_node_struct* child = node->first_child; child; child = child->next_sibling)
+                {
+                        xml_node_type type = PUGI__NODETYPE(child);
 
                         if (type == node_declaration) return true;
@@ -3025 +4306 @@
         }
 
-        inline bool allow_insert_child(xml_node_type parent, xml_node_type child)
+        PUGI__FN bool is_attribute_of(xml_attribute_struct* attr, xml_node_struct* node)
+        {
+                for (xml_attribute_struct* a = node->first_attribute; a; a = a->next_attribute)
+                        if (a == attr)
+                                return true;
+
+                return false;
+        }
+
+        PUGI__FN bool allow_insert_attribute(xml_node_type parent)
+        {
+                return parent == node_element || parent == node_declaration;
+        }
+
+        PUGI__FN bool allow_insert_child(xml_node_type parent, xml_node_type child)
         {
                 if (parent != node_document && parent != node_element) return false;
@@ -3034 +4329 @@
         }
 
-        void recursive_copy_skip(xml_node& dest, const xml_node& source, const xml_node& skip)
-        {
-                assert(dest.type() == source.type());
-
-                switch (source.type())
-                {
-                case node_element:
-                {
-                        dest.set_name(source.name());
-
-                        for (xml_attribute a = source.first_attribute(); a; a = a.next_attribute())
-                                dest.append_attribute(a.name()).set_value(a.value());
-
-                        for (xml_node c = source.first_child(); c; c = c.next_sibling())
-                        {
-                                if (c == skip) continue;
-
-                                xml_node cc = dest.append_child(c.type());
-                                assert(cc);
-
-                                recursive_copy_skip(cc, c, skip);
-                        }
-
-                        break;
-                }
-
-                case node_pcdata:
-                case node_cdata:
-                case node_comment:
-        case node_doctype:
-                        dest.set_value(source.value());
-                        break;
-
-                case node_pi:
-                        dest.set_name(source.name());
-                        dest.set_value(source.value());
-                        break;
-
-                case node_declaration:
-                {
-                        dest.set_name(source.name());
-
-                        for (xml_attribute a = source.first_attribute(); a; a = a.next_attribute())
-                                dest.append_attribute(a.name()).set_value(a.value());
-
-                        break;
-                }
-
-                default:
-                        assert(!"Invalid node type");
-                }
+        PUGI__FN bool allow_move(xml_node parent, xml_node child)
+        {
+                // check that child can be a child of parent
+                if (!allow_insert_child(parent.type(), child.type()))
+                        return false;
+
+                // check that node is not moved between documents
+                if (parent.root() != child.root())
+                        return false;
+
+                // check that new parent is not in the child subtree
+                xml_node cur = parent;
+
+                while (cur)
+                {
+                        if (cur == child)
+                                return false;
+
+                        cur = cur.parent();
+                }
+
+                return true;
+        }
+
+        template <typename String, typename Header>
+        PUGI__FN void node_copy_string(String& dest, Header& header, uintptr_t header_mask, char_t* source, Header& source_header, xml_allocator* alloc)
+        {
+                assert(!dest && (header & header_mask) == 0);
+
+                if (source)
+                {
+                        if (alloc && (source_header & header_mask) == 0)
+                        {
+                                dest = source;
+
+                                // since strcpy_insitu can reuse document buffer memory we need to mark both source and dest as shared
+                                header |= xml_memory_page_contents_shared_mask;
+                                source_header |= xml_memory_page_contents_shared_mask;
+                        }
+                        else
+                                strcpy_insitu(dest, header, header_mask, source, strlength(source));
+                }
+        }
+
+        PUGI__FN void node_copy_contents(xml_node_struct* dn, xml_node_struct* sn, xml_allocator* shared_alloc)
+        {
+                node_copy_string(dn->name, dn->header, xml_memory_page_name_allocated_mask, sn->name, sn->header, shared_alloc);
+                node_copy_string(dn->value, dn->header, xml_memory_page_value_allocated_mask, sn->value, sn->header, shared_alloc);
+
+                for (xml_attribute_struct* sa = sn->first_attribute; sa; sa = sa->next_attribute)
+                {
+                        xml_attribute_struct* da = append_new_attribute(dn, get_allocator(dn));
+
+                        if (da)
+                        {
+                                node_copy_string(da->name, da->header, xml_memory_page_name_allocated_mask, sa->name, sa->header, shared_alloc);
+                                node_copy_string(da->value, da->header, xml_memory_page_value_allocated_mask, sa->value, sa->header, shared_alloc);
+                        }
+                }
+        }
+
+        PUGI__FN void node_copy_tree(xml_node_struct* dn, xml_node_struct* sn)
+        {
+                xml_allocator& alloc = get_allocator(dn);
+                xml_allocator* shared_alloc = (&alloc == &get_allocator(sn)) ? &alloc : 0;
+
+                node_copy_contents(dn, sn, shared_alloc);
+
+                xml_node_struct* dit = dn;
+                xml_node_struct* sit = sn->first_child;
+
+                while (sit && sit != sn)
+                {
+                        if (sit != dn)
+                        {
+                                xml_node_struct* copy = append_new_node(dit, alloc, PUGI__NODETYPE(sit));
+
+                                if (copy)
+                                {
+                                        node_copy_contents(copy, sit, shared_alloc);
+
+                                        if (sit->first_child)
+                                        {
+                                                dit = copy;
+                                                sit = sit->first_child;
+                                                continue;
+                                        }
+                                }
+                        }
+
+                        // continue to the next node
+                        do
+                        {
+                                if (sit->next_sibling)
+                                {
+                                        sit = sit->next_sibling;
+                                        break;
+                                }
+
+                                sit = sit->parent;
+                                dit = dit->parent;
+                        }
+                        while (sit != sn);
+                }
+        }
+
+        PUGI__FN void node_copy_attribute(xml_attribute_struct* da, xml_attribute_struct* sa)
+        {
+                xml_allocator& alloc = get_allocator(da);
+                xml_allocator* shared_alloc = (&alloc == &get_allocator(sa)) ? &alloc : 0;
+
+                node_copy_string(da->name, da->header, xml_memory_page_name_allocated_mask, sa->name, sa->header, shared_alloc);
+                node_copy_string(da->value, da->header, xml_memory_page_value_allocated_mask, sa->value, sa->header, shared_alloc);
+        }
+
+        inline bool is_text_node(xml_node_struct* node)
+        {
+                xml_node_type type = PUGI__NODETYPE(node);
+
+                return type == node_pcdata || type == node_cdata;
+        }
+
+        // get value with conversion functions
+        template <typename U> U string_to_integer(const char_t* value, U minneg, U maxpos)
+        {
+                U result = 0;
+                const char_t* s = value;
+
+                while (PUGI__IS_CHARTYPE(*s, ct_space))
+                        s++;
+
+                bool negative = (*s == '-');
+
+                s += (*s == '+' || *s == '-');
+
+                bool overflow = false;
+
+                if (s[0] == '0' && (s[1] | ' ') == 'x')
+                {
+                        s += 2;
+
+                        // since overflow detection relies on length of the sequence skip leading zeros
+                        while (*s == '0')
+                                s++;
+
+                        const char_t* start = s;
+
+                        for (;;)
+                        {
+                                if (static_cast<unsigned>(*s - '0') < 10)
+                                        result = result * 16 + (*s - '0');
+                                else if (static_cast<unsigned>((*s | ' ') - 'a') < 6)
+                                        result = result * 16 + ((*s | ' ') - 'a' + 10);
+                                else
+                                        break;
+
+                                s++;
+                        }
+
+                        size_t digits = static_cast<size_t>(s - start);
+
+                        overflow = digits > sizeof(U) * 2;
+                }
+                else
+                {
+                        // since overflow detection relies on length of the sequence skip leading zeros
+                        while (*s == '0')
+                                s++;
+
+                        const char_t* start = s;
+
+                        for (;;)
+                        {
+                                if (static_cast<unsigned>(*s - '0') < 10)
+                                        result = result * 10 + (*s - '0');
+                                else
+                                        break;
+
+                                s++;
+                        }
+
+                        size_t digits = static_cast<size_t>(s - start);
+
+                        PUGI__STATIC_ASSERT(sizeof(U) == 8 || sizeof(U) == 4 || sizeof(U) == 2);
+
+                        const size_t max_digits10 = sizeof(U) == 8 ? 20 : sizeof(U) == 4 ? 10 : 5;
+                        const char_t max_lead = sizeof(U) == 8 ? '1' : sizeof(U) == 4 ? '4' : '6';
+                        const size_t high_bit = sizeof(U) * 8 - 1;
+
+                        overflow = digits >= max_digits10 && !(digits == max_digits10 && (*start < max_lead || (*start == max_lead && result >> high_bit)));
+                }
+
+                if (negative)
+                        return (overflow || result > minneg) ? 0 - minneg : 0 - result;
+                else
+                        return (overflow || result > maxpos) ? maxpos : result;
+        }
+
+        PUGI__FN int get_value_int(const char_t* value)
+        {
+                return string_to_integer<unsigned int>(value, 0 - static_cast<unsigned int>(INT_MIN), INT_MAX);
+        }
+
+        PUGI__FN unsigned int get_value_uint(const char_t* value)
+        {
+                return string_to_integer<unsigned int>(value, 0, UINT_MAX);
+        }
+
+        PUGI__FN double get_value_double(const char_t* value)
+        {
+        #ifdef PUGIXML_WCHAR_MODE
+                return wcstod(value, 0);
+        #else
+                return strtod(value, 0);
+        #endif
+        }
+
+        PUGI__FN float get_value_float(const char_t* value)
+        {
+        #ifdef PUGIXML_WCHAR_MODE
+                return static_cast<float>(wcstod(value, 0));
+        #else
+                return static_cast<float>(strtod(value, 0));
+        #endif
+        }
+
+        PUGI__FN bool get_value_bool(const char_t* value)
+        {
+                // only look at first char
+                char_t first = *value;
+
+                // 1*, t* (true), T* (True), y* (yes), Y* (YES)
+                return (first == '1' || first == 't' || first == 'T' || first == 'y' || first == 'Y');
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN long long get_value_llong(const char_t* value)
+        {
+                return string_to_integer<unsigned long long>(value, 0 - static_cast<unsigned long long>(LLONG_MIN), LLONG_MAX);
+        }
+
+        PUGI__FN unsigned long long get_value_ullong(const char_t* value)
+        {
+                return string_to_integer<unsigned long long>(value, 0, ULLONG_MAX);
+        }
+#endif
+
+        template <typename U> PUGI__FN char_t* integer_to_string(char_t* begin, char_t* end, U value, bool negative)
+        {
+                char_t* result = end - 1;
+                U rest = negative ? 0 - value : value;
+
+                do
+                {
+                        *result-- = static_cast<char_t>('0' + (rest % 10));
+                        rest /= 10;
+                }
+                while (rest);
+
+                assert(result >= begin);
+                (void)begin;
+
+                *result = '-';
+
+                return result + !negative;
+        }
+
+        // set value with conversion functions
+        template <typename String, typename Header>
+        PUGI__FN bool set_value_ascii(String& dest, Header& header, uintptr_t header_mask, char* buf)
+        {
+        #ifdef PUGIXML_WCHAR_MODE
+                char_t wbuf[128];
+                assert(strlen(buf) < sizeof(wbuf) / sizeof(wbuf[0]));
+
+                size_t offset = 0;
+                for (; buf[offset]; ++offset) wbuf[offset] = buf[offset];
+
+                return strcpy_insitu(dest, header, header_mask, wbuf, offset);
+        #else
+                return strcpy_insitu(dest, header, header_mask, buf, strlen(buf));
+        #endif
+        }
+
+        template <typename U, typename String, typename Header>
+        PUGI__FN bool set_value_integer(String& dest, Header& header, uintptr_t header_mask, U value, bool negative)
+        {
+                char_t buf[64];
+                char_t* end = buf + sizeof(buf) / sizeof(buf[0]);
+                char_t* begin = integer_to_string(buf, end, value, negative);
+
+                return strcpy_insitu(dest, header, header_mask, begin, end - begin);
+        }
+
+        template <typename String, typename Header>
+        PUGI__FN bool set_value_convert(String& dest, Header& header, uintptr_t header_mask, float value)
+        {
+                char buf[128];
+                sprintf(buf, "%.9g", value);
+
+                return set_value_ascii(dest, header, header_mask, buf);
+        }
+
+        template <typename String, typename Header>
+        PUGI__FN bool set_value_convert(String& dest, Header& header, uintptr_t header_mask, double value)
+        {
+                char buf[128];
+                sprintf(buf, "%.17g", value);
+
+                return set_value_ascii(dest, header, header_mask, buf);
+        }
+
+        template <typename String, typename Header>
+        PUGI__FN bool set_value_bool(String& dest, Header& header, uintptr_t header_mask, bool value)
+        {
+                return strcpy_insitu(dest, header, header_mask, value ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"), value ? 4 : 5);
+        }
+
+        PUGI__FN xml_parse_result load_buffer_impl(xml_document_struct* doc, xml_node_struct* root, void* contents, size_t size, unsigned int options, xml_encoding encoding, bool is_mutable, bool own, char_t** out_buffer)
+        {
+                // check input buffer
+                if (!contents && size) return make_parse_result(status_io_error);
+
+                // get actual encoding
+                xml_encoding buffer_encoding = impl::get_buffer_encoding(encoding, contents, size);
+
+                // get private buffer
+                char_t* buffer = 0;
+                size_t length = 0;
+
+                if (!impl::convert_buffer(buffer, length, buffer_encoding, contents, size, is_mutable)) return impl::make_parse_result(status_out_of_memory);
+
+                // delete original buffer if we performed a conversion
+                if (own && buffer != contents && contents) impl::xml_memory::deallocate(contents);
+
+                // grab onto buffer if it's our buffer, user is responsible for deallocating contents himself
+                if (own || buffer != contents) *out_buffer = buffer;
+
+                // store buffer for offset_debug
+                doc->buffer = buffer;
+
+                // parse
+                xml_parse_result res = impl::xml_parser::parse(buffer, length, doc, root, options);
+
+                // remember encoding
+                res.encoding = buffer_encoding;
+
+                return res;
         }
 
         // we need to get length of entire file to load it in memory; the only (relatively) sane way to do it is via seek/tell trick
-        xml_parse_status get_file_size(FILE* file, size_t& out_result)
-        {
-        #if defined(_MSC_VER) && _MSC_VER >= 1400
+        PUGI__FN xml_parse_status get_file_size(FILE* file, size_t& out_result)
+        {
+        #if defined(PUGI__MSVC_CRT_VERSION) && PUGI__MSVC_CRT_VERSION >= 1400 && !defined(_WIN32_WCE)
                 // there are 64-bit versions of fseek/ftell, let's use them
                 typedef __int64 length_type;
@@ -3097 +4689 @@
                 length_type length = _ftelli64(file);
                 _fseeki64(file, 0, SEEK_SET);
-        #elif defined(__MINGW32__) && !defined(__NO_MINGW_LFS) && !defined(__STRICT_ANSI__)
+        #elif defined(__MINGW32__) && !defined(__NO_MINGW_LFS) && (!defined(__STRICT_ANSI__) || defined(__MINGW64_VERSION_MAJOR))
                 // there are 64-bit versions of fseek/ftell, let's use them
                 typedef off64_t length_type;
@@ -3115 +4707 @@
                 // check for I/O errors
                 if (length < 0) return status_io_error;
-                
+
                 // check for overflow
                 size_t result = static_cast<size_t>(length);
@@ -3127 +4719 @@
         }
 
-        xml_parse_result load_file_impl(xml_document& doc, FILE* file, unsigned int options, xml_encoding encoding)
+        // This function assumes that buffer has extra sizeof(char_t) writable bytes after size
+        PUGI__FN size_t zero_terminate_buffer(void* buffer, size_t size, xml_encoding encoding)
+        {
+                // We only need to zero-terminate if encoding conversion does not do it for us
+        #ifdef PUGIXML_WCHAR_MODE
+                xml_encoding wchar_encoding = get_wchar_encoding();
+
+                if (encoding == wchar_encoding || need_endian_swap_utf(encoding, wchar_encoding))
+                {
+                        size_t length = size / sizeof(char_t);
+
+                        static_cast<char_t*>(buffer)[length] = 0;
+                        return (length + 1) * sizeof(char_t);
+                }
+        #else
+                if (encoding == encoding_utf8)
+                {
+                        static_cast<char*>(buffer)[size] = 0;
+                        return size + 1;
+                }
+        #endif
+
+                return size;
+        }
+
+        PUGI__FN xml_parse_result load_file_impl(xml_document_struct* doc, FILE* file, unsigned int options, xml_encoding encoding, char_t** out_buffer)
         {
                 if (!file) return make_parse_result(status_file_not_found);
@@ -3134 +4751 @@
                 size_t size = 0;
                 xml_parse_status size_status = get_file_size(file, size);
-
-                if (size_status != status_ok)
-                {
-                        fclose(file);
-                        return make_parse_result(size_status);
-                }
-                
+                if (size_status != status_ok) return make_parse_result(size_status);
+
+                size_t max_suffix_size = sizeof(char_t);
+
                 // allocate buffer for the whole file
-                char* contents = static_cast<char*>(global_allocate(size > 0 ? size : 1));
-
-                if (!contents)
-                {
-                        fclose(file);
-                        return make_parse_result(status_out_of_memory);
-                }
+                char* contents = static_cast<char*>(xml_memory::allocate(size + max_suffix_size));
+                if (!contents) return make_parse_result(status_out_of_memory);
 
                 // read file in memory
                 size_t read_size = fread(contents, 1, size, file);
+
+                if (read_size != size)
+                {
+                        xml_memory::deallocate(contents);
+                        return make_parse_result(status_io_error);
+                }
+
+                xml_encoding real_encoding = get_buffer_encoding(encoding, contents, size);
+
+                return load_buffer_impl(doc, doc, contents, zero_terminate_buffer(contents, size, real_encoding), options, real_encoding, true, true, out_buffer);
+        }
+
+        PUGI__FN void close_file(FILE* file)
+        {
                 fclose(file);
-
-                if (read_size != size)
-                {
-                        global_deallocate(contents);
-                        return make_parse_result(status_io_error);
-                }
-                
-                return doc.load_buffer_inplace_own(contents, size, options, encoding);
         }
 
 #ifndef PUGIXML_NO_STL
-        template <typename T> xml_parse_result load_stream_impl(xml_document& doc, std::basic_istream<T>& stream, unsigned int options, xml_encoding encoding)
+        template <typename T> struct xml_stream_chunk
+        {
+                static xml_stream_chunk* create()
+                {
+                        void* memory = xml_memory::allocate(sizeof(xml_stream_chunk));
+                        if (!memory) return 0;
+
+                        return new (memory) xml_stream_chunk();
+                }
+
+                static void destroy(xml_stream_chunk* chunk)
+                {
+                        // free chunk chain
+                        while (chunk)
+                        {
+                                xml_stream_chunk* next_ = chunk->next;
+
+                                xml_memory::deallocate(chunk);
+
+                                chunk = next_;
+                        }
+                }
+
+                xml_stream_chunk(): next(0), size(0)
+                {
+                }
+
+                xml_stream_chunk* next;
+                size_t size;
+
+                T data[xml_memory_page_size / sizeof(T)];
+        };
+
+        template <typename T> PUGI__FN xml_parse_status load_stream_data_noseek(std::basic_istream<T>& stream, void** out_buffer, size_t* out_size)
+        {
+                auto_deleter<xml_stream_chunk<T> > chunks(0, xml_stream_chunk<T>::destroy);
+
+                // read file to a chunk list
+                size_t total = 0;
+                xml_stream_chunk<T>* last = 0;
+
+                while (!stream.eof())
+                {
+                        // allocate new chunk
+                        xml_stream_chunk<T>* chunk = xml_stream_chunk<T>::create();
+                        if (!chunk) return status_out_of_memory;
+
+                        // append chunk to list
+                        if (last) last = last->next = chunk;
+                        else chunks.data = last = chunk;
+
+                        // read data to chunk
+                        stream.read(chunk->data, static_cast<std::streamsize>(sizeof(chunk->data) / sizeof(T)));
+                        chunk->size = static_cast<size_t>(stream.gcount()) * sizeof(T);
+
+                        // read may set failbit | eofbit in case gcount() is less than read length, so check for other I/O errors
+                        if (stream.bad() || (!stream.eof() && stream.fail())) return status_io_error;
+
+                        // guard against huge files (chunk size is small enough to make this overflow check work)
+                        if (total + chunk->size < total) return status_out_of_memory;
+                        total += chunk->size;
+                }
+
+                size_t max_suffix_size = sizeof(char_t);
+
+                // copy chunk list to a contiguous buffer
+                char* buffer = static_cast<char*>(xml_memory::allocate(total + max_suffix_size));
+                if (!buffer) return status_out_of_memory;
+
+                char* write = buffer;
+
+                for (xml_stream_chunk<T>* chunk = chunks.data; chunk; chunk = chunk->next)
+                {
+                        assert(write + chunk->size <= buffer + total);
+                        memcpy(write, chunk->data, chunk->size);
+                        write += chunk->size;
+                }
+
+                assert(write == buffer + total);
+
+                // return buffer
+                *out_buffer = buffer;
+                *out_size = total;
+
+                return status_ok;
+        }
+
+        template <typename T> PUGI__FN xml_parse_status load_stream_data_seek(std::basic_istream<T>& stream, void** out_buffer, size_t* out_size)
         {
                 // get length of remaining data in stream
@@ -3172 +4874 @@
                 stream.seekg(pos);
 
-                if (stream.fail() || pos < 0) return make_parse_result(status_io_error);
+                if (stream.fail() || pos < 0) return status_io_error;
 
                 // guard against huge files
                 size_t read_length = static_cast<size_t>(length);
 
-                if (static_cast<std::streamsize>(read_length) != length || length < 0) return make_parse_result(status_out_of_memory);
+                if (static_cast<std::streamsize>(read_length) != length || length < 0) return status_out_of_memory;
+
+                size_t max_suffix_size = sizeof(char_t);
 
                 // read stream data into memory (guard against stream exceptions with buffer holder)
-                buffer_holder buffer(global_allocate((read_length > 0 ? read_length : 1) * sizeof(T)), global_deallocate);
-                if (!buffer.data) return make_parse_result(status_out_of_memory);
+                auto_deleter<void> buffer(xml_memory::allocate(read_length * sizeof(T) + max_suffix_size), xml_memory::deallocate);
+                if (!buffer.data) return status_out_of_memory;
 
                 stream.read(static_cast<T*>(buffer.data), static_cast<std::streamsize>(read_length));
 
                 // read may set failbit | eofbit in case gcount() is less than read_length (i.e. line ending conversion), so check for other I/O errors
-                if (stream.bad()) return make_parse_result(status_io_error);
-
-                // load data from buffer
+                if (stream.bad() || (!stream.eof() && stream.fail())) return status_io_error;
+
+                // return buffer
                 size_t actual_length = static_cast<size_t>(stream.gcount());
                 assert(actual_length <= read_length);
 
-                return doc.load_buffer_inplace_own(buffer.release(), actual_length * sizeof(T), options, encoding);
+                *out_buffer = buffer.release();
+                *out_size = actual_length * sizeof(T);
+
+                return status_ok;
+        }
+
+        template <typename T> PUGI__FN xml_parse_result load_stream_impl(xml_document_struct* doc, std::basic_istream<T>& stream, unsigned int options, xml_encoding encoding, char_t** out_buffer)
+        {
+                void* buffer = 0;
+                size_t size = 0;
+                xml_parse_status status = status_ok;
+
+                // if stream has an error bit set, bail out (otherwise tellg() can fail and we'll clear error bits)
+                if (stream.fail()) return make_parse_result(status_io_error);
+
+                // load stream to memory (using seek-based implementation if possible, since it's faster and takes less memory)
+                if (stream.tellg() < 0)
+                {
+                        stream.clear(); // clear error flags that could be set by a failing tellg
+                        status = load_stream_data_noseek(stream, &buffer, &size);
+                }
+                else
+                        status = load_stream_data_seek(stream, &buffer, &size);
+
+                if (status != status_ok) return make_parse_result(status);
+
+                xml_encoding real_encoding = get_buffer_encoding(encoding, buffer, size);
+
+                return load_buffer_impl(doc, doc, buffer, zero_terminate_buffer(buffer, size, real_encoding), options, real_encoding, true, true, out_buffer);
         }
 #endif
 
-#if defined(_MSC_VER) || defined(__BORLANDC__) || defined(__MINGW32__)
-        FILE* open_file_wide(const wchar_t* path, const wchar_t* mode)
+#if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__) || (defined(__MINGW32__) && (!defined(__STRICT_ANSI__) || defined(__MINGW64_VERSION_MAJOR)))
+        PUGI__FN FILE* open_file_wide(const wchar_t* path, const wchar_t* mode)
         {
                 return _wfopen(path, mode);
         }
 #else
-        char* convert_path_heap(const wchar_t* str)
+        PUGI__FN char* convert_path_heap(const wchar_t* str)
         {
                 assert(str);
 
                 // first pass: get length in utf8 characters
-                size_t length = wcslen(str);
-        size_t size = as_utf8_begin(str, length);
+                size_t length = strlength_wide(str);
+                size_t size = as_utf8_begin(str, length);
 
                 // allocate resulting string
-                char* result = static_cast<char*>(global_allocate(size + 1));
+                char* result = static_cast<char*>(xml_memory::allocate(size + 1));
                 if (!result) return 0;
 
                 // second pass: convert to utf8
-        as_utf8_end(result, size, str, length);
-
-                return result;
-        }
-
-        FILE* open_file_wide(const wchar_t* path, const wchar_t* mode)
+                as_utf8_end(result, size, str, length);
+
+                // zero-terminate
+                result[size] = 0;
+
+                return result;
+        }
+
+        PUGI__FN FILE* open_file_wide(const wchar_t* path, const wchar_t* mode)
         {
                 // there is no standard function to open wide paths, so our best bet is to try utf8 path
@@ -3234 +4969 @@
 
                 // free dummy buffer
-                global_deallocate(path_utf8);
+                xml_memory::deallocate(path_utf8);
 
                 return result;
         }
 #endif
-}
+
+        PUGI__FN bool save_file_impl(const xml_document& doc, FILE* file, const char_t* indent, unsigned int flags, xml_encoding encoding)
+        {
+                if (!file) return false;
+
+                xml_writer_file writer(file);
+                doc.save(writer, indent, flags, encoding);
+
+                return ferror(file) == 0;
+        }
+
+        struct name_null_sentry
+        {
+                xml_node_struct* node;
+                char_t* name;
+
+                name_null_sentry(xml_node_struct* node_): node(node_), name(node_->name)
+                {
+                        node->name = 0;
+                }
+
+                ~name_null_sentry()
+                {
+                        node->name = name;
+                }
+        };
+PUGI__NS_END
 
 namespace pugi
 {
-        xml_writer_file::xml_writer_file(void* file): file(file)
-        {
-        }
-
-        void xml_writer_file::write(const void* data, size_t size)
-        {
-                fwrite(data, size, 1, static_cast<FILE*>(file));
+        PUGI__FN xml_writer_file::xml_writer_file(void* file_): file(file_)
+        {
+        }
+
+        PUGI__FN void xml_writer_file::write(const void* data, size_t size)
+        {
+                size_t result = fwrite(data, 1, size, static_cast<FILE*>(file));
+                (void)!result; // unfortunately we can't do proper error handling here
         }
 
 #ifndef PUGIXML_NO_STL
-        xml_writer_stream::xml_writer_stream(std::basic_ostream<char, std::char_traits<char> >& stream): narrow_stream(&stream), wide_stream(0)
-        {
-        }
-
-        xml_writer_stream::xml_writer_stream(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream): narrow_stream(0), wide_stream(&stream)
-        {
-        }
-
-        void xml_writer_stream::write(const void* data, size_t size)
+        PUGI__FN xml_writer_stream::xml_writer_stream(std::basic_ostream<char, std::char_traits<char> >& stream): narrow_stream(&stream), wide_stream(0)
+        {
+        }
+
+        PUGI__FN xml_writer_stream::xml_writer_stream(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream): narrow_stream(0), wide_stream(&stream)
+        {
+        }
+
+        PUGI__FN void xml_writer_stream::write(const void* data, size_t size)
         {
                 if (narrow_stream)
@@ -3278 +5040 @@
 #endif
 
-        xml_tree_walker::xml_tree_walker(): _depth(0)
-        {
-        }
-        
-        xml_tree_walker::~xml_tree_walker()
-        {
-        }
-
-        int xml_tree_walker::depth() const
+        PUGI__FN xml_tree_walker::xml_tree_walker(): _depth(0)
+        {
+        }
+
+        PUGI__FN xml_tree_walker::~xml_tree_walker()
+        {
+        }
+
+        PUGI__FN int xml_tree_walker::depth() const
         {
                 return _depth;
         }
 
-        bool xml_tree_walker::begin(xml_node&)
+        PUGI__FN bool xml_tree_walker::begin(xml_node&)
         {
                 return true;
         }
 
-        bool xml_tree_walker::end(xml_node&)
+        PUGI__FN bool xml_tree_walker::end(xml_node&)
         {
                 return true;
         }
 
-        xml_attribute::xml_attribute(): _attr(0)
-        {
-        }
-
-        xml_attribute::xml_attribute(xml_attribute_struct* attr): _attr(attr)
-        {
-        }
-
-        xml_attribute::operator xml_attribute::unspecified_bool_type() const
-        {
-        return _attr ? &xml_attribute::_attr : 0;
-        }
-
-        bool xml_attribute::operator!() const
-        {
-                return !_attr;
-        }
-
-        bool xml_attribute::operator==(const xml_attribute& r) const
+        PUGI__FN xml_attribute::xml_attribute(): _attr(0)
+        {
+        }
+
+        PUGI__FN xml_attribute::xml_attribute(xml_attribute_struct* attr): _attr(attr)
+        {
+        }
+
+        PUGI__FN static void unspecified_bool_xml_attribute(xml_attribute***)
+        {
+        }
+
+        PUGI__FN xml_attribute::operator xml_attribute::unspecified_bool_type() const
+        {
+                return _attr ? unspecified_bool_xml_attribute : 0;
+        }
+
+        PUGI__FN bool xml_attribute::operator!() const
+        {
+                return !_attr;
+        }
+
+        PUGI__FN bool xml_attribute::operator==(const xml_attribute& r) const
         {
                 return (_attr == r._attr);
         }
-        
-        bool xml_attribute::operator!=(const xml_attribute& r) const
+
+        PUGI__FN bool xml_attribute::operator!=(const xml_attribute& r) const
         {
                 return (_attr != r._attr);
         }
 
-        bool xml_attribute::operator<(const xml_attribute& r) const
+        PUGI__FN bool xml_attribute::operator<(const xml_attribute& r) const
         {
                 return (_attr < r._attr);
         }
-        
-        bool xml_attribute::operator>(const xml_attribute& r) const
+
+        PUGI__FN bool xml_attribute::operator>(const xml_attribute& r) const
         {
                 return (_attr > r._attr);
         }
-        
-        bool xml_attribute::operator<=(const xml_attribute& r) const
+
+        PUGI__FN bool xml_attribute::operator<=(const xml_attribute& r) const
         {
                 return (_attr <= r._attr);
         }
-        
-        bool xml_attribute::operator>=(const xml_attribute& r) const
+
+        PUGI__FN bool xml_attribute::operator>=(const xml_attribute& r) const
         {
                 return (_attr >= r._attr);
         }
 
-        xml_attribute xml_attribute::next_attribute() const
-        {
-        return _attr ? xml_attribute(_attr->next_attribute) : xml_attribute();
-        }
-
-    xml_attribute xml_attribute::previous_attribute() const
-    {
-        return _attr && _attr->prev_attribute_c->next_attribute ? xml_attribute(_attr->prev_attribute_c) : xml_attribute();
-    }
-
-        int xml_attribute::as_int() const
-        {
-                if (!_attr || !_attr->value) return 0;
-
-        #ifdef PUGIXML_WCHAR_MODE
-                return (int)wcstol(_attr->value, 0, 10);
-        #else
-                return (int)strtol(_attr->value, 0, 10);
-        #endif
-        }
-
-        unsigned int xml_attribute::as_uint() const
-        {
-                if (!_attr || !_attr->value) return 0;
-
-        #ifdef PUGIXML_WCHAR_MODE
-                return (unsigned int)wcstoul(_attr->value, 0, 10);
-        #else
-                return (unsigned int)strtoul(_attr->value, 0, 10);
-        #endif
-        }
-
-        double xml_attribute::as_double() const
-        {
-                if (!_attr || !_attr->value) return 0;
-
-        #ifdef PUGIXML_WCHAR_MODE
-                return wcstod(_attr->value, 0);
-        #else
-                return strtod(_attr->value, 0);
-        #endif
-        }
-
-        float xml_attribute::as_float() const
-        {
-                if (!_attr || !_attr->value) return 0;
-
-        #ifdef PUGIXML_WCHAR_MODE
-                return (float)wcstod(_attr->value, 0);
-        #else
-                return (float)strtod(_attr->value, 0);
-        #endif
-        }
-
-        bool xml_attribute::as_bool() const
-        {
-                if (!_attr || !_attr->value) return false;
-
-                // only look at first char
-                char_t first = *_attr->value;
-
-                // 1*, t* (true), T* (True), y* (yes), Y* (YES)
-                return (first == '1' || first == 't' || first == 'T' || first == 'y' || first == 'Y');
-        }
-
-        bool xml_attribute::empty() const
+        PUGI__FN xml_attribute xml_attribute::next_attribute() const
+        {
+                return _attr ? xml_attribute(_attr->next_attribute) : xml_attribute();
+        }
+
+        PUGI__FN xml_attribute xml_attribute::previous_attribute() const
+        {
+                return _attr && _attr->prev_attribute_c->next_attribute ? xml_attribute(_attr->prev_attribute_c) : xml_attribute();
+        }
+
+        PUGI__FN const char_t* xml_attribute::as_string(const char_t* def) const
+        {
+                return (_attr && _attr->value) ? _attr->value + 0 : def;
+        }
+
+        PUGI__FN int xml_attribute::as_int(int def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_int(_attr->value) : def;
+        }
+
+        PUGI__FN unsigned int xml_attribute::as_uint(unsigned int def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_uint(_attr->value) : def;
+        }
+
+        PUGI__FN double xml_attribute::as_double(double def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_double(_attr->value) : def;
+        }
+
+        PUGI__FN float xml_attribute::as_float(float def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_float(_attr->value) : def;
+        }
+
+        PUGI__FN bool xml_attribute::as_bool(bool def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_bool(_attr->value) : def;
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN long long xml_attribute::as_llong(long long def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_llong(_attr->value) : def;
+        }
+
+        PUGI__FN unsigned long long xml_attribute::as_ullong(unsigned long long def) const
+        {
+                return (_attr && _attr->value) ? impl::get_value_ullong(_attr->value) : def;
+        }
+#endif
+
+        PUGI__FN bool xml_attribute::empty() const
         {
                 return !_attr;
         }
 
-        const char_t* xml_attribute::name() const
-        {
-                return (_attr && _attr->name) ? _attr->name : PUGIXML_TEXT("");
-        }
-
-        const char_t* xml_attribute::value() const
-        {
-                return (_attr && _attr->value) ? _attr->value : PUGIXML_TEXT("");
-        }
-
-    size_t xml_attribute::hash_value() const
-    {
-        return static_cast<size_t>(reinterpret_cast<uintptr_t>(_attr) / sizeof(xml_attribute_struct));
-    }
-
-        xml_attribute_struct* xml_attribute::internal_object() const
-        {
-        return _attr;
-        }
-
-        xml_attribute& xml_attribute::operator=(const char_t* rhs)
+        PUGI__FN const char_t* xml_attribute::name() const
+        {
+                return (_attr && _attr->name) ? _attr->name + 0 : PUGIXML_TEXT("");
+        }
+
+        PUGI__FN const char_t* xml_attribute::value() const
+        {
+                return (_attr && _attr->value) ? _attr->value + 0 : PUGIXML_TEXT("");
+        }
+
+        PUGI__FN size_t xml_attribute::hash_value() const
+        {
+                return static_cast<size_t>(reinterpret_cast<uintptr_t>(_attr) / sizeof(xml_attribute_struct));
+        }
+
+        PUGI__FN xml_attribute_struct* xml_attribute::internal_object() const
+        {
+                return _attr;
+        }
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(const char_t* rhs)
         {
                 set_value(rhs);
                 return *this;
         }
-        
-        xml_attribute& xml_attribute::operator=(int rhs)
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(int rhs)
         {
                 set_value(rhs);
@@ -3451 +5204 @@
         }
 
-        xml_attribute& xml_attribute::operator=(unsigned int rhs)
+        PUGI__FN xml_attribute& xml_attribute::operator=(unsigned int rhs)
         {
                 set_value(rhs);
@@ -3457 +5210 @@
         }
 
-        xml_attribute& xml_attribute::operator=(double rhs)
+        PUGI__FN xml_attribute& xml_attribute::operator=(long rhs)
         {
                 set_value(rhs);
                 return *this;
         }
-        
-        xml_attribute& xml_attribute::operator=(bool rhs)
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(unsigned long rhs)
         {
                 set_value(rhs);
@@ -3469 +5222 @@
         }
 
-        bool xml_attribute::set_name(const char_t* rhs)
+        PUGI__FN xml_attribute& xml_attribute::operator=(double rhs)
+        {
+                set_value(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(float rhs)
+        {
+                set_value(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(bool rhs)
+        {
+                set_value(rhs);
+                return *this;
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN xml_attribute& xml_attribute::operator=(long long rhs)
+        {
+                set_value(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_attribute& xml_attribute::operator=(unsigned long long rhs)
+        {
+                set_value(rhs);
+                return *this;
+        }
+#endif
+
+        PUGI__FN bool xml_attribute::set_name(const char_t* rhs)
         {
                 if (!_attr) return false;
-                
-                return strcpy_insitu(_attr->name, _attr->header, xml_memory_page_name_allocated_mask, rhs);
-        }
-                
-        bool xml_attribute::set_value(const char_t* rhs)
+
+                return impl::strcpy_insitu(_attr->name, _attr->header, impl::xml_memory_page_name_allocated_mask, rhs, impl::strlength(rhs));
+        }
+
+        PUGI__FN bool xml_attribute::set_value(const char_t* rhs)
         {
                 if (!_attr) return false;
 
-                return strcpy_insitu(_attr->value, _attr->header, xml_memory_page_value_allocated_mask, rhs);
-        }
-
-        bool xml_attribute::set_value(int rhs)
-        {
-                char buf[128];
-                sprintf(buf, "%d", rhs);
-        
-        #ifdef PUGIXML_WCHAR_MODE
-                char_t wbuf[128];
-                widen_ascii(wbuf, buf);
-
-                return set_value(wbuf);
-        #else
-                return set_value(buf);
-        #endif
-        }
-
-        bool xml_attribute::set_value(unsigned int rhs)
-        {
-                char buf[128];
-                sprintf(buf, "%u", rhs);
-
-        #ifdef PUGIXML_WCHAR_MODE
-                char_t wbuf[128];
-                widen_ascii(wbuf, buf);
-
-                return set_value(wbuf);
-        #else
-                return set_value(buf);
-        #endif
-        }
-
-        bool xml_attribute::set_value(double rhs)
-        {
-                char buf[128];
-                sprintf(buf, "%g", rhs);
-
-        #ifdef PUGIXML_WCHAR_MODE
-                char_t wbuf[128];
-                widen_ascii(wbuf, buf);
-
-                return set_value(wbuf);
-        #else
-                return set_value(buf);
-        #endif
-        }
-        
-        bool xml_attribute::set_value(bool rhs)
-        {
-                return set_value(rhs ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"));
-        }
+                return impl::strcpy_insitu(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, impl::strlength(rhs));
+        }
+
+        PUGI__FN bool xml_attribute::set_value(int rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned int>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(unsigned int rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned int>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, false);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(long rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned long>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(unsigned long rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned long>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, false);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(double rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(float rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_convert(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(bool rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_bool(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs);
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN bool xml_attribute::set_value(long long rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned long long>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0);
+        }
+
+        PUGI__FN bool xml_attribute::set_value(unsigned long long rhs)
+        {
+                if (!_attr) return false;
+
+                return impl::set_value_integer<unsigned long long>(_attr->value, _attr->header, impl::xml_memory_page_value_allocated_mask, rhs, false);
+        }
+#endif
 
 #ifdef __BORLANDC__
-        bool operator&&(const xml_attribute& lhs, bool rhs)
+        PUGI__FN bool operator&&(const xml_attribute& lhs, bool rhs)
         {
                 return (bool)lhs && rhs;
         }
 
-        bool operator||(const xml_attribute& lhs, bool rhs)
+        PUGI__FN bool operator||(const xml_attribute& lhs, bool rhs)
         {
                 return (bool)lhs || rhs;
@@ -3545 +5345 @@
 #endif
 
-        xml_node::xml_node(): _root(0)
-        {
-        }
-
-        xml_node::xml_node(xml_node_struct* p): _root(p)
-        {
-        }
-        
-        xml_node::operator xml_node::unspecified_bool_type() const
-        {
-        return _root ? &xml_node::_root : 0;
-        }
-
-        bool xml_node::operator!() const
-        {
-                return !_root;
-        }
-
-        xml_node::iterator xml_node::begin() const
-        {
-                return iterator(_root ? _root->first_child : 0, _root);
-        }
-
-        xml_node::iterator xml_node::end() const
+        PUGI__FN xml_node::xml_node(): _root(0)
+        {
+        }
+
+        PUGI__FN xml_node::xml_node(xml_node_struct* p): _root(p)
+        {
+        }
+
+        PUGI__FN static void unspecified_bool_xml_node(xml_node***)
+        {
+        }
+
+        PUGI__FN xml_node::operator xml_node::unspecified_bool_type() const
+        {
+                return _root ? unspecified_bool_xml_node : 0;
+        }
+
+        PUGI__FN bool xml_node::operator!() const
+        {
+                return !_root;
+        }
+
+        PUGI__FN xml_node::iterator xml_node::begin() const
+        {
+                return iterator(_root ? _root->first_child + 0 : 0, _root);
+        }
+
+        PUGI__FN xml_node::iterator xml_node::end() const
         {
                 return iterator(0, _root);
         }
-        
-        xml_node::attribute_iterator xml_node::attributes_begin() const
-        {
-                return attribute_iterator(_root ? _root->first_attribute : 0, _root);
-        }
-
-        xml_node::attribute_iterator xml_node::attributes_end() const
+
+        PUGI__FN xml_node::attribute_iterator xml_node::attributes_begin() const
+        {
+                return attribute_iterator(_root ? _root->first_attribute + 0 : 0, _root);
+        }
+
+        PUGI__FN xml_node::attribute_iterator xml_node::attributes_end() const
         {
                 return attribute_iterator(0, _root);
         }
 
-        bool xml_node::operator==(const xml_node& r) const
+        PUGI__FN xml_object_range<xml_node_iterator> xml_node::children() const
+        {
+                return xml_object_range<xml_node_iterator>(begin(), end());
+        }
+
+        PUGI__FN xml_object_range<xml_named_node_iterator> xml_node::children(const char_t* name_) const
+        {
+                return xml_object_range<xml_named_node_iterator>(xml_named_node_iterator(child(name_)._root, _root, name_), xml_named_node_iterator(0, _root, name_));
+        }
+
+        PUGI__FN xml_object_range<xml_attribute_iterator> xml_node::attributes() const
+        {
+                return xml_object_range<xml_attribute_iterator>(attributes_begin(), attributes_end());
+        }
+
+        PUGI__FN bool xml_node::operator==(const xml_node& r) const
         {
                 return (_root == r._root);
         }
 
-        bool xml_node::operator!=(const xml_node& r) const
+        PUGI__FN bool xml_node::operator!=(const xml_node& r) const
         {
                 return (_root != r._root);
         }
 
-        bool xml_node::operator<(const xml_node& r) const
+        PUGI__FN bool xml_node::operator<(const xml_node& r) const
         {
                 return (_root < r._root);
         }
-        
-        bool xml_node::operator>(const xml_node& r) const
+
+        PUGI__FN bool xml_node::operator>(const xml_node& r) const
         {
                 return (_root > r._root);
         }
-        
-        bool xml_node::operator<=(const xml_node& r) const
+
+        PUGI__FN bool xml_node::operator<=(const xml_node& r) const
         {
                 return (_root <= r._root);
         }
-        
-        bool xml_node::operator>=(const xml_node& r) const
+
+        PUGI__FN bool xml_node::operator>=(const xml_node& r) const
         {
                 return (_root >= r._root);
         }
 
-        bool xml_node::empty() const
+        PUGI__FN bool xml_node::empty() const
         {
                 return !_root;
         }
-        
-        const char_t* xml_node::name() const
-        {
-                return (_root && _root->name) ? _root->name : PUGIXML_TEXT("");
-        }
-
-        xml_node_type xml_node::type() const
-        {
-                return _root ? static_cast<xml_node_type>((_root->header & xml_memory_page_type_mask) + 1) : node_null;
-        }
-        
-        const char_t* xml_node::value() const
-        {
-                return (_root && _root->value) ? _root->value : PUGIXML_TEXT("");
-        }
-        
-        xml_node xml_node::child(const char_t* name) const
+
+        PUGI__FN const char_t* xml_node::name() const
+        {
+                return (_root && _root->name) ? _root->name + 0 : PUGIXML_TEXT("");
+        }
+
+        PUGI__FN xml_node_type xml_node::type() const
+        {
+                return _root ? PUGI__NODETYPE(_root) : node_null;
+        }
+
+        PUGI__FN const char_t* xml_node::value() const
+        {
+                return (_root && _root->value) ? _root->value + 0 : PUGIXML_TEXT("");
+        }
+
+        PUGI__FN xml_node xml_node::child(const char_t* name_) const
         {
                 if (!_root) return xml_node();
 
                 for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
-                        if (i->name && strequal(name, i->name)) return xml_node(i);
+                        if (i->name && impl::strequal(name_, i->name)) return xml_node(i);
 
                 return xml_node();
         }
 
-        xml_attribute xml_node::attribute(const char_t* name) const
+        PUGI__FN xml_attribute xml_node::attribute(const char_t* name_) const
         {
                 if (!_root) return xml_attribute();
 
                 for (xml_attribute_struct* i = _root->first_attribute; i; i = i->next_attribute)
-                        if (i->name && strequal(name, i->name))
+                        if (i->name && impl::strequal(name_, i->name))
                                 return xml_attribute(i);
-                
+
                 return xml_attribute();
         }
-        
-        xml_node xml_node::next_sibling(const char_t* name) const
+
+        PUGI__FN xml_node xml_node::next_sibling(const char_t* name_) const
         {
                 if (!_root) return xml_node();
-                
+
                 for (xml_node_struct* i = _root->next_sibling; i; i = i->next_sibling)
-                        if (i->name && strequal(name, i->name)) return xml_node(i);
+                        if (i->name && impl::strequal(name_, i->name)) return xml_node(i);
 
                 return xml_node();
         }
 
-        xml_node xml_node::next_sibling() const
+        PUGI__FN xml_node xml_node::next_sibling() const
+        {
+                return _root ? xml_node(_root->next_sibling) : xml_node();
+        }
+
+        PUGI__FN xml_node xml_node::previous_sibling(const char_t* name_) const
         {
                 if (!_root) return xml_node();
-                
-                if (_root->next_sibling) return xml_node(_root->next_sibling);
-                else return xml_node();
-        }
-
-        xml_node xml_node::previous_sibling(const char_t* name) const
+
+                for (xml_node_struct* i = _root->prev_sibling_c; i->next_sibling; i = i->prev_sibling_c)
+                        if (i->name && impl::strequal(name_, i->name)) return xml_node(i);
+
+                return xml_node();
+        }
+
+        PUGI__FN xml_attribute xml_node::attribute(const char_t* name_, xml_attribute& hint_) const
+        {
+                xml_attribute_struct* hint = hint_._attr;
+
+                // if hint is not an attribute of node, behavior is not defined
+                assert(!hint || (_root && impl::is_attribute_of(hint, _root)));
+
+                if (!_root) return xml_attribute();
+
+                // optimistically search from hint up until the end
+                for (xml_attribute_struct* i = hint; i; i = i->next_attribute)
+                        if (i->name && impl::strequal(name_, i->name))
+                        {
+                                // update hint to maximize efficiency of searching for consecutive attributes
+                                hint_._attr = i->next_attribute;
+
+                                return xml_attribute(i);
+                        }
+
+                // wrap around and search from the first attribute until the hint
+                // 'j' null pointer check is technically redundant, but it prevents a crash in case the assertion above fails
+                for (xml_attribute_struct* j = _root->first_attribute; j && j != hint; j = j->next_attribute)
+                        if (j->name && impl::strequal(name_, j->name))
+                        {
+                                // update hint to maximize efficiency of searching for consecutive attributes
+                                hint_._attr = j->next_attribute;
+
+                                return xml_attribute(j);
+                        }
+
+                return xml_attribute();
+        }
+
+        PUGI__FN xml_node xml_node::previous_sibling() const
         {
                 if (!_root) return xml_node();
-                
-                for (xml_node_struct* i = _root->prev_sibling_c; i->next_sibling; i = i->prev_sibling_c)
-                        if (i->name && strequal(name, i->name)) return xml_node(i);
-
-                return xml_node();
-        }
-
-        xml_node xml_node::previous_sibling() const
-        {
-                if (!_root) return xml_node();
-                
+
                 if (_root->prev_sibling_c->next_sibling) return xml_node(_root->prev_sibling_c);
                 else return xml_node();
         }
 
-        xml_node xml_node::parent() const
+        PUGI__FN xml_node xml_node::parent() const
         {
                 return _root ? xml_node(_root->parent) : xml_node();
         }
 
-        xml_node xml_node::root() const
+        PUGI__FN xml_node xml_node::root() const
+        {
+                return _root ? xml_node(&impl::get_document(_root)) : xml_node();
+        }
+
+        PUGI__FN xml_text xml_node::text() const
+        {
+                return xml_text(_root);
+        }
+
+        PUGI__FN const char_t* xml_node::child_value() const
+        {
+                if (!_root) return PUGIXML_TEXT("");
+
+                // element nodes can have value if parse_embed_pcdata was used
+                if (PUGI__NODETYPE(_root) == node_element && _root->value)
+                        return _root->value;
+
+                for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
+                        if (impl::is_text_node(i) && i->value)
+                                return i->value;
+
+                return PUGIXML_TEXT("");
+        }
+
+        PUGI__FN const char_t* xml_node::child_value(const char_t* name_) const
+        {
+                return child(name_).child_value();
+        }
+
+        PUGI__FN xml_attribute xml_node::first_attribute() const
+        {
+                return _root ? xml_attribute(_root->first_attribute) : xml_attribute();
+        }
+
+        PUGI__FN xml_attribute xml_node::last_attribute() const
+        {
+                return _root && _root->first_attribute ? xml_attribute(_root->first_attribute->prev_attribute_c) : xml_attribute();
+        }
+
+        PUGI__FN xml_node xml_node::first_child() const
+        {
+                return _root ? xml_node(_root->first_child) : xml_node();
+        }
+
+        PUGI__FN xml_node xml_node::last_child() const
+        {
+                return _root && _root->first_child ? xml_node(_root->first_child->prev_sibling_c) : xml_node();
+        }
+
+        PUGI__FN bool xml_node::set_name(const char_t* rhs)
+        {
+                xml_node_type type_ = _root ? PUGI__NODETYPE(_root) : node_null;
+
+                if (type_ != node_element && type_ != node_pi && type_ != node_declaration)
+                        return false;
+
+                return impl::strcpy_insitu(_root->name, _root->header, impl::xml_memory_page_name_allocated_mask, rhs, impl::strlength(rhs));
+        }
+
+        PUGI__FN bool xml_node::set_value(const char_t* rhs)
+        {
+                xml_node_type type_ = _root ? PUGI__NODETYPE(_root) : node_null;
+
+                if (type_ != node_pcdata && type_ != node_cdata && type_ != node_comment && type_ != node_pi && type_ != node_doctype)
+                        return false;
+
+                return impl::strcpy_insitu(_root->value, _root->header, impl::xml_memory_page_value_allocated_mask, rhs, impl::strlength(rhs));
+        }
+
+        PUGI__FN xml_attribute xml_node::append_attribute(const char_t* name_)
+        {
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::append_attribute(a._attr, _root);
+
+                a.set_name(name_);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::prepend_attribute(const char_t* name_)
+        {
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::prepend_attribute(a._attr, _root);
+
+                a.set_name(name_);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::insert_attribute_after(const char_t* name_, const xml_attribute& attr)
+        {
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+                if (!attr || !impl::is_attribute_of(attr._attr, _root)) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::insert_attribute_after(a._attr, attr._attr, _root);
+
+                a.set_name(name_);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::insert_attribute_before(const char_t* name_, const xml_attribute& attr)
+        {
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+                if (!attr || !impl::is_attribute_of(attr._attr, _root)) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::insert_attribute_before(a._attr, attr._attr, _root);
+
+                a.set_name(name_);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::append_copy(const xml_attribute& proto)
+        {
+                if (!proto) return xml_attribute();
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::append_attribute(a._attr, _root);
+                impl::node_copy_attribute(a._attr, proto._attr);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::prepend_copy(const xml_attribute& proto)
+        {
+                if (!proto) return xml_attribute();
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::prepend_attribute(a._attr, _root);
+                impl::node_copy_attribute(a._attr, proto._attr);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::insert_copy_after(const xml_attribute& proto, const xml_attribute& attr)
+        {
+                if (!proto) return xml_attribute();
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+                if (!attr || !impl::is_attribute_of(attr._attr, _root)) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::insert_attribute_after(a._attr, attr._attr, _root);
+                impl::node_copy_attribute(a._attr, proto._attr);
+
+                return a;
+        }
+
+        PUGI__FN xml_attribute xml_node::insert_copy_before(const xml_attribute& proto, const xml_attribute& attr)
+        {
+                if (!proto) return xml_attribute();
+                if (!impl::allow_insert_attribute(type())) return xml_attribute();
+                if (!attr || !impl::is_attribute_of(attr._attr, _root)) return xml_attribute();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_attribute();
+
+                xml_attribute a(impl::allocate_attribute(alloc));
+                if (!a) return xml_attribute();
+
+                impl::insert_attribute_before(a._attr, attr._attr, _root);
+                impl::node_copy_attribute(a._attr, proto._attr);
+
+                return a;
+        }
+
+        PUGI__FN xml_node xml_node::append_child(xml_node_type type_)
+        {
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::append_node(n._root, _root);
+
+                if (type_ == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::prepend_child(xml_node_type type_)
+        {
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::prepend_node(n._root, _root);
+
+                if (type_ == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::insert_child_before(xml_node_type type_, const xml_node& node)
+        {
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::insert_node_before(n._root, node._root);
+
+                if (type_ == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::insert_child_after(xml_node_type type_, const xml_node& node)
+        {
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::insert_node_after(n._root, node._root);
+
+                if (type_ == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::append_child(const char_t* name_)
+        {
+                xml_node result = append_child(node_element);
+
+                result.set_name(name_);
+
+                return result;
+        }
+
+        PUGI__FN xml_node xml_node::prepend_child(const char_t* name_)
+        {
+                xml_node result = prepend_child(node_element);
+
+                result.set_name(name_);
+
+                return result;
+        }
+
+        PUGI__FN xml_node xml_node::insert_child_after(const char_t* name_, const xml_node& node)
+        {
+                xml_node result = insert_child_after(node_element, node);
+
+                result.set_name(name_);
+
+                return result;
+        }
+
+        PUGI__FN xml_node xml_node::insert_child_before(const char_t* name_, const xml_node& node)
+        {
+                xml_node result = insert_child_before(node_element, node);
+
+                result.set_name(name_);
+
+                return result;
+        }
+
+        PUGI__FN xml_node xml_node::append_copy(const xml_node& proto)
+        {
+                xml_node_type type_ = proto.type();
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::append_node(n._root, _root);
+                impl::node_copy_tree(n._root, proto._root);
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::prepend_copy(const xml_node& proto)
+        {
+                xml_node_type type_ = proto.type();
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::prepend_node(n._root, _root);
+                impl::node_copy_tree(n._root, proto._root);
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::insert_copy_after(const xml_node& proto, const xml_node& node)
+        {
+                xml_node_type type_ = proto.type();
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::insert_node_after(n._root, node._root);
+                impl::node_copy_tree(n._root, proto._root);
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::insert_copy_before(const xml_node& proto, const xml_node& node)
+        {
+                xml_node_type type_ = proto.type();
+                if (!impl::allow_insert_child(type(), type_)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                xml_node n(impl::allocate_node(alloc, type_));
+                if (!n) return xml_node();
+
+                impl::insert_node_before(n._root, node._root);
+                impl::node_copy_tree(n._root, proto._root);
+
+                return n;
+        }
+
+        PUGI__FN xml_node xml_node::append_move(const xml_node& moved)
+        {
+                if (!impl::allow_move(*this, moved)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                // disable document_buffer_order optimization since moving nodes around changes document order without changing buffer pointers
+                impl::get_document(_root).header |= impl::xml_memory_page_contents_shared_mask;
+
+                impl::remove_node(moved._root);
+                impl::append_node(moved._root, _root);
+
+                return moved;
+        }
+
+        PUGI__FN xml_node xml_node::prepend_move(const xml_node& moved)
+        {
+                if (!impl::allow_move(*this, moved)) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                // disable document_buffer_order optimization since moving nodes around changes document order without changing buffer pointers
+                impl::get_document(_root).header |= impl::xml_memory_page_contents_shared_mask;
+
+                impl::remove_node(moved._root);
+                impl::prepend_node(moved._root, _root);
+
+                return moved;
+        }
+
+        PUGI__FN xml_node xml_node::insert_move_after(const xml_node& moved, const xml_node& node)
+        {
+                if (!impl::allow_move(*this, moved)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+                if (moved._root == node._root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                // disable document_buffer_order optimization since moving nodes around changes document order without changing buffer pointers
+                impl::get_document(_root).header |= impl::xml_memory_page_contents_shared_mask;
+
+                impl::remove_node(moved._root);
+                impl::insert_node_after(moved._root, node._root);
+
+                return moved;
+        }
+
+        PUGI__FN xml_node xml_node::insert_move_before(const xml_node& moved, const xml_node& node)
+        {
+                if (!impl::allow_move(*this, moved)) return xml_node();
+                if (!node._root || node._root->parent != _root) return xml_node();
+                if (moved._root == node._root) return xml_node();
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return xml_node();
+
+                // disable document_buffer_order optimization since moving nodes around changes document order without changing buffer pointers
+                impl::get_document(_root).header |= impl::xml_memory_page_contents_shared_mask;
+
+                impl::remove_node(moved._root);
+                impl::insert_node_before(moved._root, node._root);
+
+                return moved;
+        }
+
+        PUGI__FN bool xml_node::remove_attribute(const char_t* name_)
+        {
+                return remove_attribute(attribute(name_));
+        }
+
+        PUGI__FN bool xml_node::remove_attribute(const xml_attribute& a)
+        {
+                if (!_root || !a._attr) return false;
+                if (!impl::is_attribute_of(a._attr, _root)) return false;
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return false;
+
+                impl::remove_attribute(a._attr, _root);
+                impl::destroy_attribute(a._attr, alloc);
+
+                return true;
+        }
+
+        PUGI__FN bool xml_node::remove_child(const char_t* name_)
+        {
+                return remove_child(child(name_));
+        }
+
+        PUGI__FN bool xml_node::remove_child(const xml_node& n)
+        {
+                if (!_root || !n._root || n._root->parent != _root) return false;
+
+                impl::xml_allocator& alloc = impl::get_allocator(_root);
+                if (!alloc.reserve()) return false;
+
+                impl::remove_node(n._root);
+                impl::destroy_node(n._root, alloc);
+
+                return true;
+        }
+
+        PUGI__FN xml_parse_result xml_node::append_buffer(const void* contents, size_t size, unsigned int options, xml_encoding encoding)
+        {
+                // append_buffer is only valid for elements/documents
+                if (!impl::allow_insert_child(type(), node_element)) return impl::make_parse_result(status_append_invalid_root);
+
+                // get document node
+                impl::xml_document_struct* doc = &impl::get_document(_root);
+
+                // disable document_buffer_order optimization since in a document with multiple buffers comparing buffer pointers does not make sense
+                doc->header |= impl::xml_memory_page_contents_shared_mask;
+
+                // get extra buffer element (we'll store the document fragment buffer there so that we can deallocate it later)
+                impl::xml_memory_page* page = 0;
+                impl::xml_extra_buffer* extra = static_cast<impl::xml_extra_buffer*>(doc->allocate_memory(sizeof(impl::xml_extra_buffer), page));
+                (void)page;
+
+                if (!extra) return impl::make_parse_result(status_out_of_memory);
+
+                // add extra buffer to the list
+                extra->buffer = 0;
+                extra->next = doc->extra_buffers;
+                doc->extra_buffers = extra;
+
+                // name of the root has to be NULL before parsing - otherwise closing node mismatches will not be detected at the top level
+                impl::name_null_sentry sentry(_root);
+
+                return impl::load_buffer_impl(doc, _root, const_cast<void*>(contents), size, options, encoding, false, false, &extra->buffer);
+        }
+
+        PUGI__FN xml_node xml_node::find_child_by_attribute(const char_t* name_, const char_t* attr_name, const char_t* attr_value) const
         {
                 if (!_root) return xml_node();
 
-                xml_memory_page* page = reinterpret_cast<xml_memory_page*>(_root->header & xml_memory_page_pointer_mask);
-
-                return xml_node(static_cast<xml_document_struct*>(page->allocator));
-        }
-
-        const char_t* xml_node::child_value() const
-        {
-                if (!_root) return PUGIXML_TEXT("");
-                
                 for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
-                {
-                        xml_node_type type = static_cast<xml_node_type>((i->header & xml_memory_page_type_mask) + 1);
-
-                        if (i->value && (type == node_pcdata || type == node_cdata))
-                                return i->value;
-                }
-
-                return PUGIXML_TEXT("");
-        }
-
-        const char_t* xml_node::child_value(const char_t* name) const
-        {
-                return child(name).child_value();
-        }
-
-        xml_attribute xml_node::first_attribute() const
-        {
-                return _root ? xml_attribute(_root->first_attribute) : xml_attribute();
-        }
-
-        xml_attribute xml_node::last_attribute() const
-        {
-                return _root && _root->first_attribute ? xml_attribute(_root->first_attribute->prev_attribute_c) : xml_attribute();
-        }
-
-        xml_node xml_node::first_child() const
-        {
-                return _root ? xml_node(_root->first_child) : xml_node();
-        }
-
-        xml_node xml_node::last_child() const
-        {
-                return _root && _root->first_child ? xml_node(_root->first_child->prev_sibling_c) : xml_node();
-        }
-
-        bool xml_node::set_name(const char_t* rhs)
-        {
-                switch (type())
-                {
-                case node_pi:
-                case node_declaration:
-                case node_element:
-                        return strcpy_insitu(_root->name, _root->header, xml_memory_page_name_allocated_mask, rhs);
-
-                default:
-                        return false;
-                }
-        }
-                
-        bool xml_node::set_value(const char_t* rhs)
-        {
-                switch (type())
-                {
-                case node_pi:
-                case node_cdata:
-                case node_pcdata:
-                case node_comment:
-        case node_doctype:
-                        return strcpy_insitu(_root->value, _root->header, xml_memory_page_value_allocated_mask, rhs);
-
-                default:
-                        return false;
-                }
-        }
-
-        xml_attribute xml_node::append_attribute(const char_t* name)
-        {
-                if (type() != node_element && type() != node_declaration) return xml_attribute();
-                
-                xml_attribute a(append_attribute_ll(_root, get_allocator(_root)));
-                a.set_name(name);
-                
-                return a;
-        }
-
-        xml_attribute xml_node::prepend_attribute(const char_t* name)
-        {
-                if (type() != node_element && type() != node_declaration) return xml_attribute();
-                
-                xml_attribute a(allocate_attribute(get_allocator(_root)));
-                if (!a) return xml_attribute();
-
-                a.set_name(name);
-                
-        xml_attribute_struct* head = _root->first_attribute;
-
-                if (head)
-        {
-            a._attr->prev_attribute_c = head->prev_attribute_c;
-            head->prev_attribute_c = a._attr;
-        }
-        else
-            a._attr->prev_attribute_c = a._attr;
-                
-                a._attr->next_attribute = head;
-        _root->first_attribute = a._attr;
-                                
-                return a;
-        }
-
-        xml_attribute xml_node::insert_attribute_before(const char_t* name, const xml_attribute& attr)
-        {
-                if ((type() != node_element && type() != node_declaration) || attr.empty()) return xml_attribute();
-                
-                // check that attribute belongs to *this
-                xml_attribute_struct* cur = attr._attr;
-
-                while (cur->prev_attribute_c->next_attribute) cur = cur->prev_attribute_c;
-
-                if (cur != _root->first_attribute) return xml_attribute();
-
-                xml_attribute a(allocate_attribute(get_allocator(_root)));
-                if (!a) return xml_attribute();
-
-                a.set_name(name);
-
-                if (attr._attr->prev_attribute_c->next_attribute)
-                        attr._attr->prev_attribute_c->next_attribute = a._attr;
-                else
-                        _root->first_attribute = a._attr;
-                
-                a._attr->prev_attribute_c = attr._attr->prev_attribute_c;
-                a._attr->next_attribute = attr._attr;
-                attr._attr->prev_attribute_c = a._attr;
-                                
-                return a;
-        }
-
-        xml_attribute xml_node::insert_attribute_after(const char_t* name, const xml_attribute& attr)
-        {
-                if ((type() != node_element && type() != node_declaration) || attr.empty()) return xml_attribute();
-                
-                // check that attribute belongs to *this
-                xml_attribute_struct* cur = attr._attr;
-
-                while (cur->prev_attribute_c->next_attribute) cur = cur->prev_attribute_c;
-
-                if (cur != _root->first_attribute) return xml_attribute();
-
-                xml_attribute a(allocate_attribute(get_allocator(_root)));
-                if (!a) return xml_attribute();
-
-                a.set_name(name);
-
-                if (attr._attr->next_attribute)
-                        attr._attr->next_attribute->prev_attribute_c = a._attr;
-                else
-                        _root->first_attribute->prev_attribute_c = a._attr;
-                
-                a._attr->next_attribute = attr._attr->next_attribute;
-                a._attr->prev_attribute_c = attr._attr;
-                attr._attr->next_attribute = a._attr;
-
-                return a;
-        }
-
-        xml_attribute xml_node::append_copy(const xml_attribute& proto)
-        {
-                if (!proto) return xml_attribute();
-
-                xml_attribute result = append_attribute(proto.name());
-                result.set_value(proto.value());
-
-                return result;
-        }
-
-        xml_attribute xml_node::prepend_copy(const xml_attribute& proto)
-        {
-                if (!proto) return xml_attribute();
-
-                xml_attribute result = prepend_attribute(proto.name());
-                result.set_value(proto.value());
-
-                return result;
-        }
-
-        xml_attribute xml_node::insert_copy_after(const xml_attribute& proto, const xml_attribute& attr)
-        {
-                if (!proto) return xml_attribute();
-
-                xml_attribute result = insert_attribute_after(proto.name(), attr);
-                result.set_value(proto.value());
-
-                return result;
-        }
-
-        xml_attribute xml_node::insert_copy_before(const xml_attribute& proto, const xml_attribute& attr)
-        {
-                if (!proto) return xml_attribute();
-
-                xml_attribute result = insert_attribute_before(proto.name(), attr);
-                result.set_value(proto.value());
-
-                return result;
-        }
-
-        xml_node xml_node::append_child(xml_node_type type)
-        {
-                if (!allow_insert_child(this->type(), type)) return xml_node();
-                
-                xml_node n(append_node(_root, get_allocator(_root), type));
-
-                if (type == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
-
-                return n;
-        }
-
-        xml_node xml_node::prepend_child(xml_node_type type)
-        {
-                if (!allow_insert_child(this->type(), type)) return xml_node();
-                
-                xml_node n(allocate_node(get_allocator(_root), type));
-                if (!n) return xml_node();
-
-        n._root->parent = _root;
-
-        xml_node_struct* head = _root->first_child;
-
-                if (head)
-        {
-            n._root->prev_sibling_c = head->prev_sibling_c;
-            head->prev_sibling_c = n._root;
-        }
-        else
-            n._root->prev_sibling_c = n._root;
-                
-                n._root->next_sibling = head;
-        _root->first_child = n._root;
-                                
-                if (type == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
-
-                return n;
-        }
-
-        xml_node xml_node::insert_child_before(xml_node_type type, const xml_node& node)
-        {
-                if (!allow_insert_child(this->type(), type)) return xml_node();
-                if (!node._root || node._root->parent != _root) return xml_node();
-        
-                xml_node n(allocate_node(get_allocator(_root), type));
-                if (!n) return xml_node();
-
-                n._root->parent = _root;
-                
-                if (node._root->prev_sibling_c->next_sibling)
-                        node._root->prev_sibling_c->next_sibling = n._root;
-                else
-                        _root->first_child = n._root;
-                
-                n._root->prev_sibling_c = node._root->prev_sibling_c;
-                n._root->next_sibling = node._root;
-                node._root->prev_sibling_c = n._root;
-
-                if (type == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
-
-                return n;
-        }
-
-        xml_node xml_node::insert_child_after(xml_node_type type, const xml_node& node)
-        {
-                if (!allow_insert_child(this->type(), type)) return xml_node();
-                if (!node._root || node._root->parent != _root) return xml_node();
-        
-                xml_node n(allocate_node(get_allocator(_root), type));
-                if (!n) return xml_node();
-
-                n._root->parent = _root;
-        
-                if (node._root->next_sibling)
-                        node._root->next_sibling->prev_sibling_c = n._root;
-                else
-                        _root->first_child->prev_sibling_c = n._root;
-                
-                n._root->next_sibling = node._root->next_sibling;
-                n._root->prev_sibling_c = node._root;
-                node._root->next_sibling = n._root;
-
-                if (type == node_declaration) n.set_name(PUGIXML_TEXT("xml"));
-
-                return n;
-        }
-
-    xml_node xml_node::append_child(const char_t* name)
-    {
-        xml_node result = append_child(node_element);
-
-        result.set_name(name);
-
-        return result;
-    }
-
-    xml_node xml_node::prepend_child(const char_t* name)
-    {
-        xml_node result = prepend_child(node_element);
-
-        result.set_name(name);
-
-        return result;
-    }
-
-    xml_node xml_node::insert_child_after(const char_t* name, const xml_node& node)
-    {
-        xml_node result = insert_child_after(node_element, node);
-
-        result.set_name(name);
-
-        return result;
-    }
-
-    xml_node xml_node::insert_child_before(const char_t* name, const xml_node& node)
-    {
-        xml_node result = insert_child_before(node_element, node);
-
-        result.set_name(name);
-
-        return result;
-    }
-
-        xml_node xml_node::append_copy(const xml_node& proto)
-        {
-                xml_node result = append_child(proto.type());
-
-                if (result) recursive_copy_skip(result, proto, result);
-
-                return result;
-        }
-
-        xml_node xml_node::prepend_copy(const xml_node& proto)
-        {
-                xml_node result = prepend_child(proto.type());
-
-                if (result) recursive_copy_skip(result, proto, result);
-
-                return result;
-        }
-
-        xml_node xml_node::insert_copy_after(const xml_node& proto, const xml_node& node)
-        {
-                xml_node result = insert_child_after(proto.type(), node);
-
-                if (result) recursive_copy_skip(result, proto, result);
-
-                return result;
-        }
-
-        xml_node xml_node::insert_copy_before(const xml_node& proto, const xml_node& node)
-        {
-                xml_node result = insert_child_before(proto.type(), node);
-
-                if (result) recursive_copy_skip(result, proto, result);
-
-                return result;
-        }
-
-        bool xml_node::remove_attribute(const char_t* name)
-        {
-                return remove_attribute(attribute(name));
-        }
-
-        bool xml_node::remove_attribute(const xml_attribute& a)
-        {
-                if (!_root || !a._attr) return false;
-
-                // check that attribute belongs to *this
-                xml_attribute_struct* attr = a._attr;
-
-                while (attr->prev_attribute_c->next_attribute) attr = attr->prev_attribute_c;
-
-                if (attr != _root->first_attribute) return false;
-
-                if (a._attr->next_attribute) a._attr->next_attribute->prev_attribute_c = a._attr->prev_attribute_c;
-                else if (_root->first_attribute) _root->first_attribute->prev_attribute_c = a._attr->prev_attribute_c;
-                
-                if (a._attr->prev_attribute_c->next_attribute) a._attr->prev_attribute_c->next_attribute = a._attr->next_attribute;
-                else _root->first_attribute = a._attr->next_attribute;
-
-                destroy_attribute(a._attr, get_allocator(_root));
-
-                return true;
-        }
-
-        bool xml_node::remove_child(const char_t* name)
-        {
-                return remove_child(child(name));
-        }
-
-        bool xml_node::remove_child(const xml_node& n)
-        {
-                if (!_root || !n._root || n._root->parent != _root) return false;
-
-                if (n._root->next_sibling) n._root->next_sibling->prev_sibling_c = n._root->prev_sibling_c;
-                else if (_root->first_child) _root->first_child->prev_sibling_c = n._root->prev_sibling_c;
-                
-                if (n._root->prev_sibling_c->next_sibling) n._root->prev_sibling_c->next_sibling = n._root->next_sibling;
-                else _root->first_child = n._root->next_sibling;
-        
-        destroy_node(n._root, get_allocator(_root));
-
-                return true;
-        }
-
-        xml_node xml_node::find_child_by_attribute(const char_t* name, const char_t* attr_name, const char_t* attr_value) const
+                        if (i->name && impl::strequal(name_, i->name))
+                        {
+                                for (xml_attribute_struct* a = i->first_attribute; a; a = a->next_attribute)
+                                        if (a->name && impl::strequal(attr_name, a->name) && impl::strequal(attr_value, a->value ? a->value + 0 : PUGIXML_TEXT("")))
+                                                return xml_node(i);
+                        }
+
+                return xml_node();
+        }
+
+        PUGI__FN xml_node xml_node::find_child_by_attribute(const char_t* attr_name, const char_t* attr_value) const
         {
                 if (!_root) return xml_node();
-                
-                for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
-                        if (i->name && strequal(name, i->name))
-                        {
-                                for (xml_attribute_struct* a = i->first_attribute; a; a = a->next_attribute)
-                                        if (strequal(attr_name, a->name) && strequal(attr_value, a->value))
-                                                return xml_node(i);
-                        }
-
-                return xml_node();
-        }
-
-        xml_node xml_node::find_child_by_attribute(const char_t* attr_name, const char_t* attr_value) const
-        {
-                if (!_root) return xml_node();
-                
+
                 for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
                         for (xml_attribute_struct* a = i->first_attribute; a; a = a->next_attribute)
-                                if (strequal(attr_name, a->name) && strequal(attr_value, a->value))
+                                if (a->name && impl::strequal(attr_name, a->name) && impl::strequal(attr_value, a->value ? a->value + 0 : PUGIXML_TEXT("")))
                                         return xml_node(i);
 
@@ -4138 +6092 @@
 
 #ifndef PUGIXML_NO_STL
-        string_t xml_node::path(char_t delimiter) const
-        {
-                string_t path;
-
-                xml_node cursor = *this; // Make a copy.
-                
-                path = cursor.name();
-
-                while (cursor.parent())
-                {
-                        cursor = cursor.parent();
-                        
-                        string_t temp = cursor.name();
-                        temp += delimiter;
-                        temp += path;
-                        path.swap(temp);
-                }
-
-                return path;
+        PUGI__FN string_t xml_node::path(char_t delimiter) const
+        {
+                if (!_root) return string_t();
+
+                size_t offset = 0;
+
+                for (xml_node_struct* i = _root; i; i = i->parent)
+                {
+                        offset += (i != _root);
+                        offset += i->name ? impl::strlength(i->name) : 0;
+                }
+
+                string_t result;
+                result.resize(offset);
+
+                for (xml_node_struct* j = _root; j; j = j->parent)
+                {
+                        if (j != _root)
+                                result[--offset] = delimiter;
+
+                        if (j->name && *j->name)
+                        {
+                                size_t length = impl::strlength(j->name);
+
+                                offset -= length;
+                                memcpy(&result[offset], j->name, length * sizeof(char_t));
+                        }
+                }
+
+                assert(offset == 0);
+
+                return result;
         }
 #endif
 
-        xml_node xml_node::first_element_by_path(const char_t* path, char_t delimiter) const
+        PUGI__FN xml_node xml_node::first_element_by_path(const char_t* path_, char_t delimiter) const
         {
                 xml_node found = *this; // Current search context.
 
-                if (!_root || !path || !path[0]) return found;
-
-                if (path[0] == delimiter)
+                if (!_root || !path_ || !path_[0]) return found;
+
+                if (path_[0] == delimiter)
                 {
                         // Absolute path; e.g. '/foo/bar'
                         found = found.root();
-                        ++path;
-                }
-
-                const char_t* path_segment = path;
+                        ++path_;
+                }
+
+                const char_t* path_segment = path_;
 
                 while (*path_segment == delimiter) ++path_segment;
@@ -4195 +6162 @@
                         for (xml_node_struct* j = found._root->first_child; j; j = j->next_sibling)
                         {
-                                if (j->name && strequalrange(j->name, path_segment, static_cast<size_t>(path_segment_end - path_segment)))
+                                if (j->name && impl::strequalrange(j->name, path_segment, static_cast<size_t>(path_segment_end - path_segment)))
                                 {
                                         xml_node subsearch = xml_node(j).first_element_by_path(next_segment, delimiter);
@@ -4207 +6174 @@
         }
 
-        bool xml_node::traverse(xml_tree_walker& walker)
+        PUGI__FN bool xml_node::traverse(xml_tree_walker& walker)
         {
                 walker._depth = -1;
-                
+
                 xml_node arg_begin = *this;
                 if (!walker.begin(arg_begin)) return false;
 
                 xml_node cur = first_child();
-                                
+
                 if (cur)
                 {
                         ++walker._depth;
 
-                        do 
+                        do
                         {
                                 xml_node arg_for_each = cur;
                                 if (!walker.for_each(arg_for_each))
                                         return false;
-                                                
+
                                 if (cur.first_child())
                                 {
@@ -4236 +6203 @@
                                 {
                                         // Borland C++ workaround
-                                        while (!cur.next_sibling() && cur != *this && (bool)cur.parent())
+                                        while (!cur.next_sibling() && cur != *this && !cur.parent().empty())
                                         {
                                                 --walker._depth;
                                                 cur = cur.parent();
                                         }
-                                                
+
                                         if (cur != *this)
                                                 cur = cur.next_sibling();
@@ -4255 +6222 @@
         }
 
-    size_t xml_node::hash_value() const
-    {
-        return static_cast<size_t>(reinterpret_cast<uintptr_t>(_root) / sizeof(xml_node_struct));
-    }
-
-        xml_node_struct* xml_node::internal_object() const
-        {
-        return _root;
-        }
-
-        void xml_node::print(xml_writer& writer, const char_t* indent, unsigned int flags, xml_encoding encoding, unsigned int depth) const
+        PUGI__FN size_t xml_node::hash_value() const
+        {
+                return static_cast<size_t>(reinterpret_cast<uintptr_t>(_root) / sizeof(xml_node_struct));
+        }
+
+        PUGI__FN xml_node_struct* xml_node::internal_object() const
+        {
+                return _root;
+        }
+
+        PUGI__FN void xml_node::print(xml_writer& writer, const char_t* indent, unsigned int flags, xml_encoding encoding, unsigned int depth) const
         {
                 if (!_root) return;
 
-                xml_buffered_writer buffered_writer(writer, encoding);
-
-                node_output(buffered_writer, *this, indent, flags, depth);
+                impl::xml_buffered_writer buffered_writer(writer, encoding);
+
+                impl::node_output(buffered_writer, _root, indent, flags, depth);
+
+                buffered_writer.flush();
         }
 
 #ifndef PUGIXML_NO_STL
-        void xml_node::print(std::basic_ostream<char, std::char_traits<char> >& stream, const char_t* indent, unsigned int flags, xml_encoding encoding, unsigned int depth) const
+        PUGI__FN void xml_node::print(std::basic_ostream<char, std::char_traits<char> >& stream, const char_t* indent, unsigned int flags, xml_encoding encoding, unsigned int depth) const
         {
                 xml_writer_stream writer(stream);
@@ -4282 +6251 @@
         }
 
-        void xml_node::print(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream, const char_t* indent, unsigned int flags, unsigned int depth) const
+        PUGI__FN void xml_node::print(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream, const char_t* indent, unsigned int flags, unsigned int depth) const
         {
                 xml_writer_stream writer(stream);
@@ -4290 +6259 @@
 #endif
 
-        ptrdiff_t xml_node::offset_debug() const
-        {
-                xml_node_struct* r = root()._root;
-
-                if (!r) return -1;
-
-                const char_t* buffer = static_cast<xml_document_struct*>(r)->buffer;
-
-                if (!buffer) return -1;
+        PUGI__FN ptrdiff_t xml_node::offset_debug() const
+        {
+                if (!_root) return -1;
+
+                impl::xml_document_struct& doc = impl::get_document(_root);
+
+                // we can determine the offset reliably only if there is exactly once parse buffer
+                if (!doc.buffer || doc.extra_buffers) return -1;
 
                 switch (type())
@@ -4308 +6276 @@
                 case node_declaration:
                 case node_pi:
-                        return (_root->header & xml_memory_page_name_allocated_mask) ? -1 : _root->name - buffer;
+                        return _root->name && (_root->header & impl::xml_memory_page_name_allocated_or_shared_mask) == 0 ? _root->name - doc.buffer : -1;
 
                 case node_pcdata:
@@ -4314 +6282 @@
                 case node_comment:
                 case node_doctype:
-                        return (_root->header & xml_memory_page_value_allocated_mask) ? -1 : _root->value - buffer;
+                        return _root->value && (_root->header & impl::xml_memory_page_value_allocated_or_shared_mask) == 0 ? _root->value - doc.buffer : -1;
 
                 default:
@@ -4322 +6290 @@
 
 #ifdef __BORLANDC__
-        bool operator&&(const xml_node& lhs, bool rhs)
+        PUGI__FN bool operator&&(const xml_node& lhs, bool rhs)
         {
                 return (bool)lhs && rhs;
         }
 
-        bool operator||(const xml_node& lhs, bool rhs)
+        PUGI__FN bool operator||(const xml_node& lhs, bool rhs)
         {
                 return (bool)lhs || rhs;
@@ -4333 +6301 @@
 #endif
 
-        xml_node_iterator::xml_node_iterator()
-        {
-        }
-
-        xml_node_iterator::xml_node_iterator(const xml_node& node): _wrap(node), _parent(node.parent())
-        {
-        }
-
-        xml_node_iterator::xml_node_iterator(xml_node_struct* ref, xml_node_struct* parent): _wrap(ref), _parent(parent)
-        {
-        }
-
-        bool xml_node_iterator::operator==(const xml_node_iterator& rhs) const
+        PUGI__FN xml_text::xml_text(xml_node_struct* root): _root(root)
+        {
+        }
+
+        PUGI__FN xml_node_struct* xml_text::_data() const
+        {
+                if (!_root || impl::is_text_node(_root)) return _root;
+
+                // element nodes can have value if parse_embed_pcdata was used
+                if (PUGI__NODETYPE(_root) == node_element && _root->value)
+                        return _root;
+
+                for (xml_node_struct* node = _root->first_child; node; node = node->next_sibling)
+                        if (impl::is_text_node(node))
+                                return node;
+
+                return 0;
+        }
+
+        PUGI__FN xml_node_struct* xml_text::_data_new()
+        {
+                xml_node_struct* d = _data();
+                if (d) return d;
+
+                return xml_node(_root).append_child(node_pcdata).internal_object();
+        }
+
+        PUGI__FN xml_text::xml_text(): _root(0)
+        {
+        }
+
+        PUGI__FN static void unspecified_bool_xml_text(xml_text***)
+        {
+        }
+
+        PUGI__FN xml_text::operator xml_text::unspecified_bool_type() const
+        {
+                return _data() ? unspecified_bool_xml_text : 0;
+        }
+
+        PUGI__FN bool xml_text::operator!() const
+        {
+                return !_data();
+        }
+
+        PUGI__FN bool xml_text::empty() const
+        {
+                return _data() == 0;
+        }
+
+        PUGI__FN const char_t* xml_text::get() const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? d->value + 0 : PUGIXML_TEXT("");
+        }
+
+        PUGI__FN const char_t* xml_text::as_string(const char_t* def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? d->value + 0 : def;
+        }
+
+        PUGI__FN int xml_text::as_int(int def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_int(d->value) : def;
+        }
+
+        PUGI__FN unsigned int xml_text::as_uint(unsigned int def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_uint(d->value) : def;
+        }
+
+        PUGI__FN double xml_text::as_double(double def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_double(d->value) : def;
+        }
+
+        PUGI__FN float xml_text::as_float(float def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_float(d->value) : def;
+        }
+
+        PUGI__FN bool xml_text::as_bool(bool def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_bool(d->value) : def;
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN long long xml_text::as_llong(long long def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_llong(d->value) : def;
+        }
+
+        PUGI__FN unsigned long long xml_text::as_ullong(unsigned long long def) const
+        {
+                xml_node_struct* d = _data();
+
+                return (d && d->value) ? impl::get_value_ullong(d->value) : def;
+        }
+#endif
+
+        PUGI__FN bool xml_text::set(const char_t* rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::strcpy_insitu(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, impl::strlength(rhs)) : false;
+        }
+
+        PUGI__FN bool xml_text::set(int rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned int>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0) : false;
+        }
+
+        PUGI__FN bool xml_text::set(unsigned int rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned int>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, false) : false;
+        }
+
+        PUGI__FN bool xml_text::set(long rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned long>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0) : false;
+        }
+
+        PUGI__FN bool xml_text::set(unsigned long rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned long>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, false) : false;
+        }
+
+        PUGI__FN bool xml_text::set(float rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
+        }
+
+        PUGI__FN bool xml_text::set(double rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_convert(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
+        }
+
+        PUGI__FN bool xml_text::set(bool rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_bool(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs) : false;
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN bool xml_text::set(long long rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned long long>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, rhs < 0) : false;
+        }
+
+        PUGI__FN bool xml_text::set(unsigned long long rhs)
+        {
+                xml_node_struct* dn = _data_new();
+
+                return dn ? impl::set_value_integer<unsigned long long>(dn->value, dn->header, impl::xml_memory_page_value_allocated_mask, rhs, false) : false;
+        }
+#endif
+
+        PUGI__FN xml_text& xml_text::operator=(const char_t* rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(int rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(unsigned int rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(long rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(unsigned long rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(double rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(float rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(bool rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+#ifdef PUGIXML_HAS_LONG_LONG
+        PUGI__FN xml_text& xml_text::operator=(long long rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+
+        PUGI__FN xml_text& xml_text::operator=(unsigned long long rhs)
+        {
+                set(rhs);
+                return *this;
+        }
+#endif
+
+        PUGI__FN xml_node xml_text::data() const
+        {
+                return xml_node(_data());
+        }
+
+#ifdef __BORLANDC__
+        PUGI__FN bool operator&&(const xml_text& lhs, bool rhs)
+        {
+                return (bool)lhs && rhs;
+        }
+
+        PUGI__FN bool operator||(const xml_text& lhs, bool rhs)
+        {
+                return (bool)lhs || rhs;
+        }
+#endif
+
+        PUGI__FN xml_node_iterator::xml_node_iterator()
+        {
+        }
+
+        PUGI__FN xml_node_iterator::xml_node_iterator(const xml_node& node): _wrap(node), _parent(node.parent())
+        {
+        }
+
+        PUGI__FN xml_node_iterator::xml_node_iterator(xml_node_struct* ref, xml_node_struct* parent): _wrap(ref), _parent(parent)
+        {
+        }
+
+        PUGI__FN bool xml_node_iterator::operator==(const xml_node_iterator& rhs) const
         {
                 return _wrap._root == rhs._wrap._root && _parent._root == rhs._parent._root;
         }
-        
-        bool xml_node_iterator::operator!=(const xml_node_iterator& rhs) const
+
+        PUGI__FN bool xml_node_iterator::operator!=(const xml_node_iterator& rhs) const
         {
                 return _wrap._root != rhs._wrap._root || _parent._root != rhs._parent._root;
         }
 
-        xml_node& xml_node_iterator::operator*()
+        PUGI__FN xml_node& xml_node_iterator::operator*() const
         {
                 assert(_wrap._root);
@@ -4361 +6595 @@
         }
 
-        xml_node* xml_node_iterator::operator->()
+        PUGI__FN xml_node* xml_node_iterator::operator->() const
         {
                 assert(_wrap._root);
-                return &_wrap;
-        }
-
-        const xml_node_iterator& xml_node_iterator::operator++()
+                return const_cast<xml_node*>(&_wrap); // BCC5 workaround
+        }
+
+        PUGI__FN const xml_node_iterator& xml_node_iterator::operator++()
         {
                 assert(_wrap._root);
@@ -4374 +6608 @@
         }
 
-        xml_node_iterator xml_node_iterator::operator++(int)
+        PUGI__FN xml_node_iterator xml_node_iterator::operator++(int)
         {
                 xml_node_iterator temp = *this;
@@ -4381 +6615 @@
         }
 
-        const xml_node_iterator& xml_node_iterator::operator--()
+        PUGI__FN const xml_node_iterator& xml_node_iterator::operator--()
         {
                 _wrap = _wrap._root ? _wrap.previous_sibling() : _parent.last_child();
@@ -4387 +6621 @@
         }
 
-        xml_node_iterator xml_node_iterator::operator--(int)
+        PUGI__FN xml_node_iterator xml_node_iterator::operator--(int)
         {
                 xml_node_iterator temp = *this;
@@ -4394 +6628 @@
         }
 
-        xml_attribute_iterator::xml_attribute_iterator()
-        {
-        }
-
-        xml_attribute_iterator::xml_attribute_iterator(const xml_attribute& attr, const xml_node& parent): _wrap(attr), _parent(parent)
-        {
-        }
-
-        xml_attribute_iterator::xml_attribute_iterator(xml_attribute_struct* ref, xml_node_struct* parent): _wrap(ref), _parent(parent)
-        {
-        }
-
-        bool xml_attribute_iterator::operator==(const xml_attribute_iterator& rhs) const
+        PUGI__FN xml_attribute_iterator::xml_attribute_iterator()
+        {
+        }
+
+        PUGI__FN xml_attribute_iterator::xml_attribute_iterator(const xml_attribute& attr, const xml_node& parent): _wrap(attr), _parent(parent)
+        {
+        }
+
+        PUGI__FN xml_attribute_iterator::xml_attribute_iterator(xml_attribute_struct* ref, xml_node_struct* parent): _wrap(ref), _parent(parent)
+        {
+        }
+
+        PUGI__FN bool xml_attribute_iterator::operator==(const xml_attribute_iterator& rhs) const
         {
                 return _wrap._attr == rhs._wrap._attr && _parent._root == rhs._parent._root;
         }
-        
-        bool xml_attribute_iterator::operator!=(const xml_attribute_iterator& rhs) const
+
+        PUGI__FN bool xml_attribute_iterator::operator!=(const xml_attribute_iterator& rhs) const
         {
                 return _wrap._attr != rhs._wrap._attr || _parent._root != rhs._parent._root;
         }
 
-        xml_attribute& xml_attribute_iterator::operator*()
+        PUGI__FN xml_attribute& xml_attribute_iterator::operator*() const
         {
                 assert(_wrap._attr);
@@ -4422 +6656 @@
         }
 
-        xml_attribute* xml_attribute_iterator::operator->()
+        PUGI__FN xml_attribute* xml_attribute_iterator::operator->() const
         {
                 assert(_wrap._attr);
-                return &_wrap;
-        }
-
-        const xml_attribute_iterator& xml_attribute_iterator::operator++()
+                return const_cast<xml_attribute*>(&_wrap); // BCC5 workaround
+        }
+
+        PUGI__FN const xml_attribute_iterator& xml_attribute_iterator::operator++()
         {
                 assert(_wrap._attr);
@@ -4435 +6669 @@
         }
 
-        xml_attribute_iterator xml_attribute_iterator::operator++(int)
+        PUGI__FN xml_attribute_iterator xml_attribute_iterator::operator++(int)
         {
                 xml_attribute_iterator temp = *this;
@@ -4442 +6676 @@
         }
 
-        const xml_attribute_iterator& xml_attribute_iterator::operator--()
+        PUGI__FN const xml_attribute_iterator& xml_attribute_iterator::operator--()
         {
                 _wrap = _wrap._attr ? _wrap.previous_attribute() : _parent.last_attribute();
@@ -4448 +6682 @@
         }
 
-        xml_attribute_iterator xml_attribute_iterator::operator--(int)
+        PUGI__FN xml_attribute_iterator xml_attribute_iterator::operator--(int)
         {
                 xml_attribute_iterator temp = *this;
@@ -4455 +6689 @@
         }
 
-    xml_parse_result::xml_parse_result(): status(status_internal_error), offset(0), encoding(encoding_auto)
-    {
-    }
-
-    xml_parse_result::operator bool() const
-    {
-        return status == status_ok;
-    }
-
-        const char* xml_parse_result::description() const
+        PUGI__FN xml_named_node_iterator::xml_named_node_iterator(): _name(0)
+        {
+        }
+
+        PUGI__FN xml_named_node_iterator::xml_named_node_iterator(const xml_node& node, const char_t* name): _wrap(node), _parent(node.parent()), _name(name)
+        {
+        }
+
+        PUGI__FN xml_named_node_iterator::xml_named_node_iterator(xml_node_struct* ref, xml_node_struct* parent, const char_t* name): _wrap(ref), _parent(parent), _name(name)
+        {
+        }
+
+        PUGI__FN bool xml_named_node_iterator::operator==(const xml_named_node_iterator& rhs) const
+        {
+                return _wrap._root == rhs._wrap._root && _parent._root == rhs._parent._root;
+        }
+
+        PUGI__FN bool xml_named_node_iterator::operator!=(const xml_named_node_iterator& rhs) const
+        {
+                return _wrap._root != rhs._wrap._root || _parent._root != rhs._parent._root;
+        }
+
+        PUGI__FN xml_node& xml_named_node_iterator::operator*() const
+        {
+                assert(_wrap._root);
+                return _wrap;
+        }
+
+        PUGI__FN xml_node* xml_named_node_iterator::operator->() const
+        {
+                assert(_wrap._root);
+                return const_cast<xml_node*>(&_wrap); // BCC5 workaround
+        }
+
+        PUGI__FN const xml_named_node_iterator& xml_named_node_iterator::operator++()
+        {
+                assert(_wrap._root);
+                _wrap = _wrap.next_sibling(_name);
+                return *this;
+        }
+
+        PUGI__FN xml_named_node_iterator xml_named_node_iterator::operator++(int)
+        {
+                xml_named_node_iterator temp = *this;
+                ++*this;
+                return temp;
+        }
+
+        PUGI__FN const xml_named_node_iterator& xml_named_node_iterator::operator--()
+        {
+                if (_wrap._root)
+                        _wrap = _wrap.previous_sibling(_name);
+                else
+                {
+                        _wrap = _parent.last_child();
+
+                        if (!impl::strequal(_wrap.name(), _name))
+                                _wrap = _wrap.previous_sibling(_name);
+                }
+
+                return *this;
+        }
+
+        PUGI__FN xml_named_node_iterator xml_named_node_iterator::operator--(int)
+        {
+                xml_named_node_iterator temp = *this;
+                --*this;
+                return temp;
+        }
+
+        PUGI__FN xml_parse_result::xml_parse_result(): status(status_internal_error), offset(0), encoding(encoding_auto)
+        {
+        }
+
+        PUGI__FN xml_parse_result::operator bool() const
+        {
+                return status == status_ok;
+        }
+
+        PUGI__FN const char* xml_parse_result::description() const
         {
                 switch (status)
@@ -4487 +6791 @@
                 case status_end_element_mismatch: return "Start-end tags mismatch";
 
+                case status_append_invalid_root: return "Unable to append nodes: root is not an element or document";
+
+                case status_no_document_element: return "No document element found";
+
                 default: return "Unknown error";
                 }
         }
 
-        xml_document::xml_document(): _buffer(0)
-        {
-                create();
-        }
-
-        xml_document::~xml_document()
-        {
-                destroy();
-        }
-
-        void xml_document::reset()
-        {
-                destroy();
-                create();
-        }
-
-    void xml_document::reset(const xml_document& proto)
-    {
-        reset();
-
-        for (xml_node cur = proto.first_child(); cur; cur = cur.next_sibling())
-            append_copy(cur);
-    }
-
-        void xml_document::create()
-        {
+        PUGI__FN xml_document::xml_document(): _buffer(0)
+        {
+                _create();
+        }
+
+        PUGI__FN xml_document::~xml_document()
+        {
+                _destroy();
+        }
+
+        PUGI__FN void xml_document::reset()
+        {
+                _destroy();
+                _create();
+        }
+
+        PUGI__FN void xml_document::reset(const xml_document& proto)
+        {
+                reset();
+
+                for (xml_node cur = proto.first_child(); cur; cur = cur.next_sibling())
+                        append_copy(cur);
+        }
+
+        PUGI__FN void xml_document::_create()
+        {
+                assert(!_root);
+
+        #ifdef PUGIXML_COMPACT
+                const size_t page_offset = sizeof(uint32_t);
+        #else
+                const size_t page_offset = 0;
+        #endif
+
                 // initialize sentinel page
-                STATIC_ASSERT(offsetof(xml_memory_page, data) + sizeof(xml_document_struct) + xml_memory_page_alignment <= sizeof(_memory));
-
-                // align upwards to page boundary
-                void* page_memory = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(_memory) + (xml_memory_page_alignment - 1)) & ~(xml_memory_page_alignment - 1));
+                PUGI__STATIC_ASSERT(sizeof(impl::xml_memory_page) + sizeof(impl::xml_document_struct) + page_offset <= sizeof(_memory));
 
                 // prepare page structure
-                xml_memory_page* page = xml_memory_page::construct(page_memory);
-
-                page->busy_size = xml_memory_page_size;
+                impl::xml_memory_page* page = impl::xml_memory_page::construct(_memory);
+                assert(page);
+
+                page->busy_size = impl::xml_memory_page_size;
+
+                // setup first page marker
+        #ifdef PUGIXML_COMPACT
+                // round-trip through void* to avoid 'cast increases required alignment of target type' warning
+                page->compact_page_marker = reinterpret_cast<uint32_t*>(static_cast<void*>(reinterpret_cast<char*>(page) + sizeof(impl::xml_memory_page)));
+                *page->compact_page_marker = sizeof(impl::xml_memory_page);
+        #endif
 
                 // allocate new root
-                _root = new (page->data) xml_document_struct(page);
+                _root = new (reinterpret_cast<char*>(page) + sizeof(impl::xml_memory_page) + page_offset) impl::xml_document_struct(page);
                 _root->prev_sibling_c = _root;
 
                 // setup sentinel page
-                page->allocator = static_cast<xml_document_struct*>(_root);
-        }
-
-        void xml_document::destroy()
-        {
+                page->allocator = static_cast<impl::xml_document_struct*>(_root);
+
+                // setup hash table pointer in allocator
+        #ifdef PUGIXML_COMPACT
+                page->allocator->_hash = &static_cast<impl::xml_document_struct*>(_root)->hash;
+        #endif
+
+                // verify the document allocation
+                assert(reinterpret_cast<char*>(_root) + sizeof(impl::xml_document_struct) <= _memory + sizeof(_memory));
+        }
+
+        PUGI__FN void xml_document::_destroy()
+        {
+                assert(_root);
+
                 // destroy static storage
                 if (_buffer)
                 {
-                        global_deallocate(_buffer);
+                        impl::xml_memory::deallocate(_buffer);
                         _buffer = 0;
                 }
 
+                // destroy extra buffers (note: no need to destroy linked list nodes, they're allocated using document allocator)
+                for (impl::xml_extra_buffer* extra = static_cast<impl::xml_document_struct*>(_root)->extra_buffers; extra; extra = extra->next)
+                {
+                        if (extra->buffer) impl::xml_memory::deallocate(extra->buffer);
+                }
+
                 // destroy dynamic storage, leave sentinel page (it's in static memory)
-                if (_root)
-                {
-                        xml_memory_page* root_page = reinterpret_cast<xml_memory_page*>(_root->header & xml_memory_page_pointer_mask);
-                        assert(root_page && !root_page->prev && !root_page->memory);
-
-                        // destroy all pages
-                        for (xml_memory_page* page = root_page->next; page; )
-                        {
-                                xml_memory_page* next = page->next;
-
-                                xml_allocator::deallocate_page(page);
-
-                                page = next;
-                        }
-
-                        // cleanup root page
-                        root_page->allocator = 0;
-                        root_page->next = 0;
-                        root_page->busy_size = root_page->freed_size = 0;
-
-                        _root = 0;
-                }
+                impl::xml_memory_page* root_page = PUGI__GETPAGE(_root);
+                assert(root_page && !root_page->prev);
+                assert(reinterpret_cast<char*>(root_page) >= _memory && reinterpret_cast<char*>(root_page) < _memory + sizeof(_memory));
+
+                for (impl::xml_memory_page* page = root_page->next; page; )
+                {
+                        impl::xml_memory_page* next = page->next;
+
+                        impl::xml_allocator::deallocate_page(page);
+
+                        page = next;
+                }
+
+        #ifdef PUGIXML_COMPACT
+                // destroy hash table
+                static_cast<impl::xml_document_struct*>(_root)->hash.clear();
+        #endif
+
+                _root = 0;
         }
 
 #ifndef PUGIXML_NO_STL
-        xml_parse_result xml_document::load(std::basic_istream<char, std::char_traits<char> >& stream, unsigned int options, xml_encoding encoding)
+        PUGI__FN xml_parse_result xml_document::load(std::basic_istream<char, std::char_traits<char> >& stream, unsigned int options, xml_encoding encoding)
         {
                 reset();
 
-                return load_stream_impl(*this, stream, options, encoding);
-        }
-
-        xml_parse_result xml_document::load(std::basic_istream<wchar_t, std::char_traits<wchar_t> >& stream, unsigned int options)
+                return impl::load_stream_impl(static_cast<impl::xml_document_struct*>(_root), stream, options, encoding, &_buffer);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load(std::basic_istream<wchar_t, std::char_traits<wchar_t> >& stream, unsigned int options)
         {
                 reset();
 
-                return load_stream_impl(*this, stream, options, encoding_wchar);
+                return impl::load_stream_impl(static_cast<impl::xml_document_struct*>(_root), stream, options, encoding_wchar, &_buffer);
         }
 #endif
 
-        xml_parse_result xml_document::load(const char_t* contents, unsigned int options)
+        PUGI__FN xml_parse_result xml_document::load_string(const char_t* contents, unsigned int options)
         {
                 // Force native encoding (skip autodetection)
@@ -4595 +6929 @@
         #endif
 
-                return load_buffer(contents, strlength(contents) * sizeof(char_t), options, encoding);
-        }
-
-        xml_parse_result xml_document::load_file(const char* path, unsigned int options, xml_encoding encoding)
+                return load_buffer(contents, impl::strlength(contents) * sizeof(char_t), options, encoding);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load(const char_t* contents, unsigned int options)
+        {
+                return load_string(contents, options);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load_file(const char* path_, unsigned int options, xml_encoding encoding)
         {
                 reset();
 
-                FILE* file = fopen(path, "rb");
-
-                return load_file_impl(*this, file, options, encoding);
-        }
-
-        xml_parse_result xml_document::load_file(const wchar_t* path, unsigned int options, xml_encoding encoding)
+                using impl::auto_deleter; // MSVC7 workaround
+                auto_deleter<FILE> file(fopen(path_, "rb"), impl::close_file);
+
+                return impl::load_file_impl(static_cast<impl::xml_document_struct*>(_root), file.data, options, encoding, &_buffer);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load_file(const wchar_t* path_, unsigned int options, xml_encoding encoding)
         {
                 reset();
 
-                FILE* file = open_file_wide(path, L"rb");
-
-                return load_file_impl(*this, file, options, encoding);
-        }
-
-        xml_parse_result xml_document::load_buffer_impl(void* contents, size_t size, unsigned int options, xml_encoding encoding, bool is_mutable, bool own)
+                using impl::auto_deleter; // MSVC7 workaround
+                auto_deleter<FILE> file(impl::open_file_wide(path_, L"rb"), impl::close_file);
+
+                return impl::load_file_impl(static_cast<impl::xml_document_struct*>(_root), file.data, options, encoding, &_buffer);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load_buffer(const void* contents, size_t size, unsigned int options, xml_encoding encoding)
         {
                 reset();
 
-                // check input buffer
-                assert(contents || size == 0);
-
-                // get actual encoding
-                xml_encoding buffer_encoding = get_buffer_encoding(encoding, contents, size);
-
-                // get private buffer
-                char_t* buffer = 0;
-                size_t length = 0;
-
-                if (!convert_buffer(buffer, length, buffer_encoding, contents, size, is_mutable)) return make_parse_result(status_out_of_memory);
-                
-                // delete original buffer if we performed a conversion
-                if (own && buffer != contents && contents) global_deallocate(contents);
-
-                // parse
-                xml_parse_result res = xml_parser::parse(buffer, length, _root, options);
-
-                // remember encoding
-                res.encoding = buffer_encoding;
-
-                // grab onto buffer if it's our buffer, user is responsible for deallocating contens himself
-                if (own || buffer != contents) _buffer = buffer;
-
-                return res;
-        }
-
-        xml_parse_result xml_document::load_buffer(const void* contents, size_t size, unsigned int options, xml_encoding encoding)
-        {
-                return load_buffer_impl(const_cast<void*>(contents), size, options, encoding, false, false);
-        }
-
-        xml_parse_result xml_document::load_buffer_inplace(void* contents, size_t size, unsigned int options, xml_encoding encoding)
-        {
-                return load_buffer_impl(contents, size, options, encoding, true, false);
-        }
-                
-        xml_parse_result xml_document::load_buffer_inplace_own(void* contents, size_t size, unsigned int options, xml_encoding encoding)
-        {
-                return load_buffer_impl(contents, size, options, encoding, true, true);
-        }
-
-        void xml_document::save(xml_writer& writer, const char_t* indent, unsigned int flags, xml_encoding encoding) const
-        {
-                if (flags & format_write_bom) write_bom(writer, get_write_encoding(encoding));
-
-                xml_buffered_writer buffered_writer(writer, encoding);
-
-                if (!(flags & format_no_declaration) && !has_declaration(*this))
-                {
-                        buffered_writer.write(PUGIXML_TEXT("<?xml version=\"1.0\"?>"));
+                return impl::load_buffer_impl(static_cast<impl::xml_document_struct*>(_root), _root, const_cast<void*>(contents), size, options, encoding, false, false, &_buffer);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load_buffer_inplace(void* contents, size_t size, unsigned int options, xml_encoding encoding)
+        {
+                reset();
+
+                return impl::load_buffer_impl(static_cast<impl::xml_document_struct*>(_root), _root, contents, size, options, encoding, true, false, &_buffer);
+        }
+
+        PUGI__FN xml_parse_result xml_document::load_buffer_inplace_own(void* contents, size_t size, unsigned int options, xml_encoding encoding)
+        {
+                reset();
+
+                return impl::load_buffer_impl(static_cast<impl::xml_document_struct*>(_root), _root, contents, size, options, encoding, true, true, &_buffer);
+        }
+
+        PUGI__FN void xml_document::save(xml_writer& writer, const char_t* indent, unsigned int flags, xml_encoding encoding) const
+        {
+                impl::xml_buffered_writer buffered_writer(writer, encoding);
+
+                if ((flags & format_write_bom) && encoding != encoding_latin1)
+                {
+                        // BOM always represents the codepoint U+FEFF, so just write it in native encoding
+                #ifdef PUGIXML_WCHAR_MODE
+                        unsigned int bom = 0xfeff;
+                        buffered_writer.write(static_cast<wchar_t>(bom));
+                #else
+                        buffered_writer.write('\xef', '\xbb', '\xbf');
+                #endif
+                }
+
+                if (!(flags & format_no_declaration) && !impl::has_declaration(_root))
+                {
+                        buffered_writer.write_string(PUGIXML_TEXT("<?xml version=\"1.0\""));
+                        if (encoding == encoding_latin1) buffered_writer.write_string(PUGIXML_TEXT(" encoding=\"ISO-8859-1\""));
+                        buffered_writer.write('?', '>');
                         if (!(flags & format_raw)) buffered_writer.write('\n');
                 }
 
-                node_output(buffered_writer, *this, indent, flags, 0);
+                impl::node_output(buffered_writer, _root, indent, flags, 0);
+
+                buffered_writer.flush();
         }
 
 #ifndef PUGIXML_NO_STL
-        void xml_document::save(std::basic_ostream<char, std::char_traits<char> >& stream, const char_t* indent, unsigned int flags, xml_encoding encoding) const
+        PUGI__FN void xml_document::save(std::basic_ostream<char, std::char_traits<char> >& stream, const char_t* indent, unsigned int flags, xml_encoding encoding) const
         {
                 xml_writer_stream writer(stream);
@@ -4685 +7014 @@
         }
 
-        void xml_document::save(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream, const char_t* indent, unsigned int flags) const
+        PUGI__FN void xml_document::save(std::basic_ostream<wchar_t, std::char_traits<wchar_t> >& stream, const char_t* indent, unsigned int flags) const
         {
                 xml_writer_stream writer(stream);
@@ -4693 +7022 @@
 #endif
 
-        bool xml_document::save_file(const char* path, const char_t* indent, unsigned int flags, xml_encoding encoding) const
-        {
-                FILE* file = fopen(path, "wb");
-                if (!file) return false;
-
-                xml_writer_file writer(file);
-                save(writer, indent, flags, encoding);
-
-                fclose(file);
-
-                return true;
-        }
-
-        bool xml_document::save_file(const wchar_t* path, const char_t* indent, unsigned int flags, xml_encoding encoding) const
-        {
-                FILE* file = open_file_wide(path, L"wb");
-                if (!file) return false;
-
-                xml_writer_file writer(file);
-                save(writer, indent, flags, encoding);
-
-                fclose(file);
-
-                return true;
-        }
-
-    xml_node xml_document::document_element() const
-    {
+        PUGI__FN bool xml_document::save_file(const char* path_, const char_t* indent, unsigned int flags, xml_encoding encoding) const
+        {
+                using impl::auto_deleter; // MSVC7 workaround
+                auto_deleter<FILE> file(fopen(path_, (flags & format_save_file_text) ? "w" : "wb"), impl::close_file);
+
+                return impl::save_file_impl(*this, file.data, indent, flags, encoding);
+        }
+
+        PUGI__FN bool xml_document::save_file(const wchar_t* path_, const char_t* indent, unsigned int flags, xml_encoding encoding) const
+        {
+                using impl::auto_deleter; // MSVC7 workaround
+                auto_deleter<FILE> file(impl::open_file_wide(path_, (flags & format_save_file_text) ? L"w" : L"wb"), impl::close_file);
+
+                return impl::save_file_impl(*this, file.data, indent, flags, encoding);
+        }
+
+        PUGI__FN xml_node xml_document::document_element() const
+        {
+                assert(_root);
+
                 for (xml_node_struct* i = _root->first_child; i; i = i->next_sibling)
-                        if ((i->header & xml_memory_page_type_mask) + 1 == node_element)
-                return xml_node(i);
-
-        return xml_node();
-    }
+                        if (PUGI__NODETYPE(i) == node_element)
+                                return xml_node(i);
+
+                return xml_node();
+        }
 
 #ifndef PUGIXML_NO_STL
-        std::string PUGIXML_FUNCTION as_utf8(const wchar_t* str)
+        PUGI__FN std::string PUGIXML_FUNCTION as_utf8(const wchar_t* str)
         {
                 assert(str);
 
-        return as_utf8_impl(str, wcslen(str));
-        }
-
-        std::string PUGIXML_FUNCTION as_utf8(const std::wstring& str)
-        {
-        return as_utf8_impl(str.c_str(), str.size());
-        }
-        
-        std::wstring PUGIXML_FUNCTION as_wide(const char* str)
+                return impl::as_utf8_impl(str, impl::strlength_wide(str));
+        }
+
+        PUGI__FN std::string PUGIXML_FUNCTION as_utf8(const std::basic_string<wchar_t>& str)
+        {
+                return impl::as_utf8_impl(str.c_str(), str.size());
+        }
+
+        PUGI__FN std::basic_string<wchar_t> PUGIXML_FUNCTION as_wide(const char* str)
         {
                 assert(str);
 
-        return as_wide_impl(str, strlen(str));
-        }
-        
-        std::wstring PUGIXML_FUNCTION as_wide(const std::string& str)
-        {
-        return as_wide_impl(str.c_str(), str.size());
+                return impl::as_wide_impl(str, strlen(str));
+        }
+
+        PUGI__FN std::basic_string<wchar_t> PUGIXML_FUNCTION as_wide(const std::string& str)
+        {
+                return impl::as_wide_impl(str.c_str(), str.size());
         }
 #endif
 
-    void PUGIXML_FUNCTION set_memory_management_functions(allocation_function allocate, deallocation_function deallocate)
-    {
-        global_allocate = allocate;
-        global_deallocate = deallocate;
-    }
-
-    allocation_function PUGIXML_FUNCTION get_memory_allocation_function()
-    {
-        return global_allocate;
-    }
-
-    deallocation_function PUGIXML_FUNCTION get_memory_deallocation_function()
-    {
-        return global_deallocate;
-    }
+        PUGI__FN void PUGIXML_FUNCTION set_memory_management_functions(allocation_function allocate, deallocation_function deallocate)
+        {
+                impl::xml_memory::allocate = allocate;
+                impl::xml_memory::deallocate = deallocate;
+        }
+
+        PUGI__FN allocation_function PUGIXML_FUNCTION get_memory_allocation_function()
+        {
+                return impl::xml_memory::allocate;
+        }
+
+        PUGI__FN deallocation_function PUGIXML_FUNCTION get_memory_deallocation_function()
+        {
+                return impl::xml_memory::deallocate;
+        }
 }
 
@@ -4775 +7096 @@
 {
         // Workarounds for (non-standard) iterator category detection for older versions (MSVC7/IC8 and earlier)
-        std::bidirectional_iterator_tag _Iter_cat(const xml_node_iterator&)
+        PUGI__FN std::bidirectional_iterator_tag _Iter_cat(const pugi::xml_node_iterator&)
         {
                 return std::bidirectional_iterator_tag();
         }
 
-        std::bidirectional_iterator_tag _Iter_cat(const xml_attribute_iterator&)
+        PUGI__FN std::bidirectional_iterator_tag _Iter_cat(const pugi::xml_attribute_iterator&)
+        {
+                return std::bidirectional_iterator_tag();
+        }
+
+        PUGI__FN std::bidirectional_iterator_tag _Iter_cat(const pugi::xml_named_node_iterator&)
         {
                 return std::bidirectional_iterator_tag();
@@ -4791 +7117 @@
 {
         // Workarounds for (non-standard) iterator category detection
-        std::bidirectional_iterator_tag __iterator_category(const xml_node_iterator&)
+        PUGI__FN std::bidirectional_iterator_tag __iterator_category(const pugi::xml_node_iterator&)
         {
                 return std::bidirectional_iterator_tag();
         }
 
-        std::bidirectional_iterator_tag __iterator_category(const xml_attribute_iterator&)
+        PUGI__FN std::bidirectional_iterator_tag __iterator_category(const pugi::xml_attribute_iterator&)
+        {
+                return std::bidirectional_iterator_tag();
+        }
+
+        PUGI__FN std::bidirectional_iterator_tag __iterator_category(const pugi::xml_named_node_iterator&)
         {
                 return std::bidirectional_iterator_tag();
@@ -4804 +7135 @@
 
 #ifndef PUGIXML_NO_XPATH
-
 // STL replacements
-namespace
-{
+PUGI__NS_BEGIN
         struct equal_to
         {
@@ -4860 +7189 @@
         template <typename I> void reverse(I begin, I end)
         {
-                while (begin + 1 < end) swap(*begin++, *--end);
+                while (end - begin > 1) swap(*begin++, *--end);
         }
 
@@ -4866 +7195 @@
         {
                 // fast skip head
-                while (begin + 1 < end && *begin != *(begin + 1)) begin++;
+                while (end - begin > 1 && *begin != *(begin + 1)) begin++;
 
                 if (begin == end) return begin;
 
                 // last written element
-                I write = begin++; 
+                I write = begin++;
 
                 // merge unique elements
@@ -5031 +7360 @@
                 if (begin != end) insertion_sort(begin, end, pred, &*begin);
         }
-}
+PUGI__NS_END
 
 // Allocator used for AST and evaluation stacks
-namespace
-{
+PUGI__NS_BEGIN
+        static const size_t xpath_memory_page_size =
+        #ifdef PUGIXML_MEMORY_XPATH_PAGE_SIZE
+                PUGIXML_MEMORY_XPATH_PAGE_SIZE
+        #else
+                4096
+        #endif
+                ;
+
+        static const uintptr_t xpath_memory_block_alignment = sizeof(double) > sizeof(void*) ? sizeof(double) : sizeof(void*);
+
         struct xpath_memory_block
-        {       
+        {
                 xpath_memory_block* next;
-
-                char data[4096];
+                size_t capacity;
+
+                union
+                {
+                        char data[xpath_memory_page_size];
+                        double alignment;
+                };
         };
-                
+
         class xpath_allocator
         {
@@ -5059 +7402 @@
                 #endif
                 }
-                
+
                 void* allocate_nothrow(size_t size)
                 {
-                        const size_t block_capacity = sizeof(_root->data);
-
-                        // align size so that we're able to store pointers in subsequent blocks
-                        size = (size + sizeof(void*) - 1) & ~(sizeof(void*) - 1);
-
-                        if (_root_size + size <= block_capacity)
-                        {
-                                void* buf = _root->data + _root_size;
+                        // round size up to block alignment boundary
+                        size = (size + xpath_memory_block_alignment - 1) & ~(xpath_memory_block_alignment - 1);
+
+                        if (_root_size + size <= _root->capacity)
+                        {
+                                void* buf = &_root->data[0] + _root_size;
                                 _root_size += size;
                                 return buf;
@@ -5075 +7416 @@
                         else
                         {
-                                size_t block_data_size = (size > block_capacity) ? size : block_capacity;
-                                size_t block_size = block_data_size + offsetof(xpath_memory_block, data);
-
-                                xpath_memory_block* block = static_cast<xpath_memory_block*>(global_allocate(block_size));
+                                // make sure we have at least 1/4th of the page free after allocation to satisfy subsequent allocation requests
+                                size_t block_capacity_base = sizeof(_root->data);
+                                size_t block_capacity_req = size + block_capacity_base / 4;
+                                size_t block_capacity = (block_capacity_base > block_capacity_req) ? block_capacity_base : block_capacity_req;
+
+                                size_t block_size = block_capacity + offsetof(xpath_memory_block, data);
+
+                                xpath_memory_block* block = static_cast<xpath_memory_block*>(xml_memory::allocate(block_size));
                                 if (!block) return 0;
-                                
+
                                 block->next = _root;
-                                
+                                block->capacity = block_capacity;
+
                                 _root = block;
                                 _root_size = size;
-                                
+
                                 return block->data;
                         }
@@ -5109 +7455 @@
                 void* reallocate(void* ptr, size_t old_size, size_t new_size)
                 {
-                        // align size so that we're able to store pointers in subsequent blocks
-                        old_size = (old_size + sizeof(void*) - 1) & ~(sizeof(void*) - 1);
-                        new_size = (new_size + sizeof(void*) - 1) & ~(sizeof(void*) - 1);
+                        // round size up to block alignment boundary
+                        old_size = (old_size + xpath_memory_block_alignment - 1) & ~(xpath_memory_block_alignment - 1);
+                        new_size = (new_size + xpath_memory_block_alignment - 1) & ~(xpath_memory_block_alignment - 1);
 
                         // we can only reallocate the last object
-                        assert(ptr == 0 || static_cast<char*>(ptr) + old_size == _root->data + _root_size);
+                        assert(ptr == 0 || static_cast<char*>(ptr) + old_size == &_root->data[0] + _root_size);
 
                         // adjust root size so that we have not allocated the object at all
@@ -5129 +7475 @@
                         {
                                 // copy old data
-                                assert(new_size > old_size);
+                                assert(new_size >= old_size);
                                 memcpy(result, ptr, old_size);
 
@@ -5143 +7489 @@
                                         {
                                                 // deallocate the whole page, unless it was the first one
-                                                global_deallocate(_root->next);
+                                                xml_memory::deallocate(_root->next);
                                                 _root->next = next;
                                         }
@@ -5161 +7507 @@
                                 xpath_memory_block* next = cur->next;
 
-                                global_deallocate(cur);
+                                xml_memory::deallocate(cur);
 
                                 cur = next;
@@ -5180 +7526 @@
                                 xpath_memory_block* next = cur->next;
 
-                                global_deallocate(cur);
+                                xml_memory::deallocate(cur);
 
                                 cur = next;
@@ -5222 +7568 @@
                 {
                         blocks[0].next = blocks[1].next = 0;
+                        blocks[0].capacity = blocks[1].capacity = sizeof(blocks[0].data);
 
                         stack.result = &result;
@@ -5237 +7584 @@
                 }
         };
-}
+PUGI__NS_END
 
 // String class
-namespace
-{
+PUGI__NS_BEGIN
         class xpath_string
         {
                 const char_t* _buffer;
                 bool _uses_heap;
+                size_t _length_heap;
 
                 static char_t* duplicate_string(const char_t* string, size_t length, xpath_allocator* alloc)
@@ -5258 +7605 @@
                 }
 
-                static char_t* duplicate_string(const char_t* string, xpath_allocator* alloc)
-                {
-                        return duplicate_string(string, strlength(string), alloc);
+                xpath_string(const char_t* buffer, bool uses_heap_, size_t length_heap): _buffer(buffer), _uses_heap(uses_heap_), _length_heap(length_heap)
+                {
                 }
 
         public:
-                xpath_string(): _buffer(PUGIXML_TEXT("")), _uses_heap(false)
-                {
-                }
-
-                explicit xpath_string(const char_t* str, xpath_allocator* alloc)
-                {
-                        bool empty = (*str == 0);
-
-                        _buffer = empty ? PUGIXML_TEXT("") : duplicate_string(str, alloc);
-                        _uses_heap = !empty;
-                }
-
-                explicit xpath_string(const char_t* str, bool use_heap): _buffer(str), _uses_heap(use_heap)
-                {
-                }
-
-                xpath_string(const char_t* begin, const char_t* end, xpath_allocator* alloc)
+                static xpath_string from_const(const char_t* str)
+                {
+                        return xpath_string(str, false, 0);
+                }
+
+                static xpath_string from_heap_preallocated(const char_t* begin, const char_t* end)
+                {
+                        assert(begin <= end && *end == 0);
+
+                        return xpath_string(begin, true, static_cast<size_t>(end - begin));
+                }
+
+                static xpath_string from_heap(const char_t* begin, const char_t* end, xpath_allocator* alloc)
                 {
                         assert(begin <= end);
 
-                        bool empty = (begin == end);
-
-                        _buffer = empty ? PUGIXML_TEXT("") : duplicate_string(begin, static_cast<size_t>(end - begin), alloc);
-                        _uses_heap = !empty;
+                        size_t length = static_cast<size_t>(end - begin);
+
+                        return length == 0 ? xpath_string() : xpath_string(duplicate_string(begin, length, alloc), true, length);
+                }
+
+                xpath_string(): _buffer(PUGIXML_TEXT("")), _uses_heap(false), _length_heap(0)
+                {
                 }
 
@@ -5303 +7648 @@
                         {
                                 // need to make heap copy
-                                size_t target_length = strlength(_buffer);
-                                size_t source_length = strlength(o._buffer);
-                                size_t length = target_length + source_length;
+                                size_t target_length = length();
+                                size_t source_length = o.length();
+                                size_t result_length = target_length + source_length;
 
                                 // allocate new buffer
-                                char_t* result = static_cast<char_t*>(alloc->reallocate(_uses_heap ? const_cast<char_t*>(_buffer) : 0, (target_length + 1) * sizeof(char_t), (length + 1) * sizeof(char_t)));
+                                char_t* result = static_cast<char_t*>(alloc->reallocate(_uses_heap ? const_cast<char_t*>(_buffer) : 0, (target_length + 1) * sizeof(char_t), (result_length + 1) * sizeof(char_t)));
                                 assert(result);
 
@@ -5316 +7661 @@
                                 // append second string to the new buffer
                                 memcpy(result + target_length, o._buffer, source_length * sizeof(char_t));
-                                result[length] = 0;
+                                result[result_length] = 0;
 
                                 // finalize
                                 _buffer = result;
                                 _uses_heap = true;
+                                _length_heap = result_length;
                         }
                 }
@@ -5331 +7677 @@
                 size_t length() const
                 {
-                        return strlength(_buffer);
-                }
-                
+                        return _uses_heap ? _length_heap : strlength(_buffer);
+                }
+
                 char_t* data(xpath_allocator* alloc)
                 {
@@ -5339 +7685 @@
                         if (!_uses_heap)
                         {
-                                _buffer = duplicate_string(_buffer, alloc);
+                                size_t length_ = strlength(_buffer);
+
+                                _buffer = duplicate_string(_buffer, length_, alloc);
                                 _uses_heap = true;
+                                _length_heap = length_;
                         }
 
@@ -5366 +7715 @@
                 }
         };
-
-        xpath_string xpath_string_const(const char_t* str)
-        {
-                return xpath_string(str, false);
-        }
-}
-
-namespace
-{
-        bool starts_with(const char_t* string, const char_t* pattern)
+PUGI__NS_END
+
+PUGI__NS_BEGIN
+        PUGI__FN bool starts_with(const char_t* string, const char_t* pattern)
         {
                 while (*pattern && *string == *pattern)
@@ -5386 +7729 @@
         }
 
-        const char_t* find_char(const char_t* s, char_t c)
+        PUGI__FN const char_t* find_char(const char_t* s, char_t c)
         {
         #ifdef PUGIXML_WCHAR_MODE
@@ -5395 +7738 @@
         }
 
-        const char_t* find_substring(const char_t* s, const char_t* p)
+        PUGI__FN const char_t* find_substring(const char_t* s, const char_t* p)
         {
         #ifdef PUGIXML_WCHAR_MODE
@@ -5406 +7749 @@
 
         // Converts symbol to lower case, if it is an ASCII one
-        char_t tolower_ascii(char_t ch)
+        PUGI__FN char_t tolower_ascii(char_t ch)
         {
                 return static_cast<unsigned int>(ch - 'A') < 26 ? static_cast<char_t>(ch | ' ') : ch;
         }
 
-        xpath_string string_value(const xpath_node& na, xpath_allocator* alloc)
+        PUGI__FN xpath_string string_value(const xpath_node& na, xpath_allocator* alloc)
         {
                 if (na.attribute())
-                        return xpath_string_const(na.attribute().value());
+                        return xpath_string::from_const(na.attribute().value());
                 else
                 {
-                        const xml_node& n = na.node();
+                        xml_node n = na.node();
 
                         switch (n.type())
@@ -5425 +7768 @@
                         case node_comment:
                         case node_pi:
-                                return xpath_string_const(n.value());
-                        
+                                return xpath_string::from_const(n.value());
+
                         case node_document:
                         case node_element:
@@ -5432 +7775 @@
                                 xpath_string result;
 
+                                // element nodes can have value if parse_embed_pcdata was used
+                                if (n.value()[0])
+                                        result.append(xpath_string::from_const(n.value()), alloc);
+
                                 xml_node cur = n.first_child();
-                                
+
                                 while (cur && cur != n)
                                 {
                                         if (cur.type() == node_pcdata || cur.type() == node_cdata)
-                                                result.append(xpath_string_const(cur.value()), alloc);
+                                                result.append(xpath_string::from_const(cur.value()), alloc);
 
                                         if (cur.first_child())
@@ -5451 +7798 @@
                                         }
                                 }
-                                
+
                                 return result;
                         }
-                        
+
                         default:
                                 return xpath_string();
@@ -5460 +7807 @@
                 }
         }
-        
-        unsigned int node_height(xml_node n)
-        {
-            unsigned int result = 0;
-            
-            while (n)
-            {
-                ++result;
-                n = n.parent();
-            }
-            
-            return result;
-        }
-        
-        bool node_is_before(xml_node ln, unsigned int lh, xml_node rn, unsigned int rh)
-        {
-                // normalize heights
-                for (unsigned int i = rh; i < lh; i++) ln = ln.parent();
-                for (unsigned int j = lh; j < rh; j++) rn = rn.parent();
-            
+
+        PUGI__FN bool node_is_before_sibling(xml_node_struct* ln, xml_node_struct* rn)
+        {
+                assert(ln->parent == rn->parent);
+
+                // there is no common ancestor (the shared parent is null), nodes are from different documents
+                if (!ln->parent) return ln < rn;
+
+                // determine sibling order
+                xml_node_struct* ls = ln;
+                xml_node_struct* rs = rn;
+
+                while (ls && rs)
+                {
+                        if (ls == rn) return true;
+                        if (rs == ln) return false;
+
+                        ls = ls->next_sibling;
+                        rs = rs->next_sibling;
+                }
+
+                // if rn sibling chain ended ln must be before rn
+                return !rs;
+        }
+
+        PUGI__FN bool node_is_before(xml_node_struct* ln, xml_node_struct* rn)
+        {
+                // find common ancestor at the same depth, if any
+                xml_node_struct* lp = ln;
+                xml_node_struct* rp = rn;
+
+                while (lp && rp && lp->parent != rp->parent)
+                {
+                        lp = lp->parent;
+                        rp = rp->parent;
+                }
+
+                // parents are the same!
+                if (lp && rp) return node_is_before_sibling(lp, rp);
+
+                // nodes are at different depths, need to normalize heights
+                bool left_higher = !lp;
+
+                while (lp)
+                {
+                        lp = lp->parent;
+                        ln = ln->parent;
+                }
+
+                while (rp)
+                {
+                        rp = rp->parent;
+                        rn = rn->parent;
+                }
+
                 // one node is the ancestor of the other
-            if (ln == rn) return lh < rh;
-            
-                // find common ancestor
-            while (ln.parent() != rn.parent())
-            {
-                ln = ln.parent();
-                rn = rn.parent();
-            }
-
-                // there is no common ancestor (the shared parent is null), nodes are from different documents
-                if (!ln.parent()) return ln < rn;
-
-                // determine sibling order
-        for (; ln; ln = ln.next_sibling())
-            if (ln == rn)
-                return true;
-                
-        return false;
-    }
-
-    bool node_is_ancestor(xml_node parent, xml_node node)
-    {
-        while (node && node != parent) node = node.parent();
-
-        return parent && node == parent;
-    }
-
-    const void* document_order(const xpath_node& xnode)
-    {
-        xml_node_struct* node = xnode.node().internal_object();
-
-        if (node)
-        {
-            if (node->name && (node->header & xml_memory_page_name_allocated_mask) == 0) return node->name;
-            if (node->value && (node->header & xml_memory_page_value_allocated_mask) == 0) return node->value;
-            return 0;
-        }
-
-        xml_attribute_struct* attr = xnode.attribute().internal_object();
-
-        if (attr)
-        {
-            if ((attr->header & xml_memory_page_name_allocated_mask) == 0) return attr->name;
-            if ((attr->header & xml_memory_page_value_allocated_mask) == 0) return attr->value;
-            return 0;
-        }
+                if (ln == rn) return left_higher;
+
+                // find common ancestor... again
+                while (ln->parent != rn->parent)
+                {
+                        ln = ln->parent;
+                        rn = rn->parent;
+                }
+
+                return node_is_before_sibling(ln, rn);
+        }
+
+        PUGI__FN bool node_is_ancestor(xml_node_struct* parent, xml_node_struct* node)
+        {
+                while (node && node != parent) node = node->parent;
+
+                return parent && node == parent;
+        }
+
+        PUGI__FN const void* document_buffer_order(const xpath_node& xnode)
+        {
+                xml_node_struct* node = xnode.node().internal_object();
+
+                if (node)
+                {
+                        if ((get_document(node).header & xml_memory_page_contents_shared_mask) == 0)
+                        {
+                                if (node->name && (node->header & impl::xml_memory_page_name_allocated_or_shared_mask) == 0) return node->name;
+                                if (node->value && (node->header & impl::xml_memory_page_value_allocated_or_shared_mask) == 0) return node->value;
+                        }
+
+                        return 0;
+                }
+
+                xml_attribute_struct* attr = xnode.attribute().internal_object();
+
+                if (attr)
+                {
+                        if ((get_document(attr).header & xml_memory_page_contents_shared_mask) == 0)
+                        {
+                                if ((attr->header & impl::xml_memory_page_name_allocated_or_shared_mask) == 0) return attr->name;
+                                if ((attr->header & impl::xml_memory_page_value_allocated_or_shared_mask) == 0) return attr->value;
+                        }
+
+                        return 0;
+                }
 
                 return 0;
-    }
-    
+        }
+
         struct document_order_comparator
         {
@@ -5536 +7918 @@
                 {
                         // optimized document order based check
-                        const void* lo = document_order(lhs);
-                        const void* ro = document_order(rhs);
+                        const void* lo = document_buffer_order(lhs);
+                        const void* ro = document_buffer_order(rhs);
 
                         if (lo && ro) return lo < ro;
 
-            // slow comparison
+                        // slow comparison
                         xml_node ln = lhs.node(), rn = rhs.node();
 
@@ -5551 +7933 @@
                                 {
                                         // determine sibling order
-                                    for (xml_attribute a = lhs.attribute(); a; a = a.next_attribute())
-                                        if (a == rhs.attribute())
-                                            return true;
-                                    
-                                    return false;
-                                }
-                                
+                                        for (xml_attribute a = lhs.attribute(); a; a = a.next_attribute())
+                                                if (a == rhs.attribute())
+                                                        return true;
+
+                                        return false;
+                                }
+
                                 // compare attribute parents
                                 ln = lhs.parent();
@@ -5566 +7948 @@
                                 // attributes go after the parent element
                                 if (lhs.parent() == rhs.node()) return false;
-                                
+
                                 ln = lhs.parent();
                         }
@@ -5573 +7955 @@
                                 // attributes go after the parent element
                                 if (rhs.parent() == lhs.node()) return true;
-                                
+
                                 rn = rhs.parent();
                         }
 
                         if (ln == rn) return false;
-                        
-                        unsigned int lh = node_height(ln);
-                        unsigned int rh = node_height(rn);
-                        
-                        return node_is_before(ln, lh, rn, rh);
+
+                        if (!ln || !rn) return ln < rn;
+
+                        return node_is_before(ln.internal_object(), rn.internal_object());
                 }
         };
@@ -5594 +7975 @@
                 }
         };
-        
-        double gen_nan()
+
+        PUGI__FN double gen_nan()
         {
         #if defined(__STDC_IEC_559__) || ((FLT_RADIX - 0 == 2) && (FLT_MAX_EXP - 0 == 128) && (FLT_MANT_DIG - 0 == 24))
-                union { float f; int32_t i; } u[sizeof(float) == sizeof(int32_t) ? 1 : -1];
-                u[0].i = 0x7fc00000;
-                return u[0].f;
+                PUGI__STATIC_ASSERT(sizeof(float) == sizeof(uint32_t));
+                typedef uint32_t UI; // BCC5 workaround
+                union { float f; UI i; } u;
+                u.i = 0x7fc00000;
+                return u.f;
         #else
                 // fallback
@@ -5607 +7990 @@
         #endif
         }
-        
-        bool is_nan(double value)
-        {
-        #if defined(_MSC_VER) || defined(__BORLANDC__)
+
+        PUGI__FN bool is_nan(double value)
+        {
+        #if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__)
                 return !!_isnan(value);
         #elif defined(fpclassify) && defined(FP_NAN)
@@ -5620 +8003 @@
         #endif
         }
-        
-        const char_t* convert_number_to_string_special(double value)
-        {
-        #if defined(_MSC_VER) || defined(__BORLANDC__)
+
+        PUGI__FN const char_t* convert_number_to_string_special(double value)
+        {
+        #if defined(PUGI__MSVC_CRT_VERSION) || defined(__BORLANDC__)
                 if (_finite(value)) return (value == 0) ? PUGIXML_TEXT("0") : 0;
                 if (_isnan(value)) return PUGIXML_TEXT("NaN");
-                return PUGIXML_TEXT("-Infinity") + (value > 0);
+                return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");
         #elif defined(fpclassify) && defined(FP_NAN) && defined(FP_INFINITE) && defined(FP_ZERO)
                 switch (fpclassify(value))
@@ -5634 +8017 @@
 
                 case FP_INFINITE:
-                        return PUGIXML_TEXT("-Infinity") + (value > 0);
+                        return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");
 
                 case FP_ZERO:
@@ -5648 +8031 @@
                 if (v == 0) return PUGIXML_TEXT("0");
                 if (v != v) return PUGIXML_TEXT("NaN");
-                if (v * 2 == v) return PUGIXML_TEXT("-Infinity") + (value > 0);
+                if (v * 2 == v) return value > 0 ? PUGIXML_TEXT("Infinity") : PUGIXML_TEXT("-Infinity");
                 return 0;
         #endif
         }
-        
-        bool convert_number_to_boolean(double value)
+
+        PUGI__FN bool convert_number_to_boolean(double value)
         {
                 return (value != 0 && !is_nan(value));
         }
-        
-        void truncate_zeros(char* begin, char* end)
+
+        PUGI__FN void truncate_zeros(char* begin, char* end)
         {
                 while (begin != end && end[-1] == '0') end--;
@@ -5666 +8049 @@
 
         // gets mantissa digits in the form of 0.xxxxx with 0. implied and the exponent
-#if defined(_MSC_VER) && _MSC_VER >= 1400
-        void convert_number_to_mantissa_exponent(double value, char* buffer, size_t buffer_size, char** out_mantissa, int* out_exponent)
+#if defined(PUGI__MSVC_CRT_VERSION) && PUGI__MSVC_CRT_VERSION >= 1400 && !defined(_WIN32_WCE)
+        PUGI__FN void convert_number_to_mantissa_exponent(double value, char* buffer, size_t buffer_size, char** out_mantissa, int* out_exponent)
         {
                 // get base values
@@ -5681 +8064 @@
         }
 #else
-        void convert_number_to_mantissa_exponent(double value, char* buffer, size_t buffer_size, char** out_mantissa, int* out_exponent)
+        PUGI__FN void convert_number_to_mantissa_exponent(double value, char* buffer, size_t buffer_size, char** out_mantissa, int* out_exponent)
         {
                 // get a scientific notation value with IEEE DBL_DIG decimals
@@ -5712 +8095 @@
 #endif
 
-        xpath_string convert_number_to_string(double value, xpath_allocator* alloc)
+        PUGI__FN xpath_string convert_number_to_string(double value, xpath_allocator* alloc)
         {
                 // try special number conversion
                 const char_t* special = convert_number_to_string_special(value);
-                if (special) return xpath_string_const(special);
+                if (special) return xpath_string::from_const(special);
 
                 // get mantissa + exponent form
-                char mantissa_buffer[64];
+                char mantissa_buffer[32];
 
                 char* mantissa;
@@ -5725 +8108 @@
                 convert_number_to_mantissa_exponent(value, mantissa_buffer, sizeof(mantissa_buffer), &mantissa, &exponent);
 
+                // allocate a buffer of suitable length for the number
+                size_t result_size = strlen(mantissa_buffer) + (exponent > 0 ? exponent : -exponent) + 4;
+                char_t* result = static_cast<char_t*>(alloc->allocate(sizeof(char_t) * result_size));
+                assert(result);
+
                 // make the number!
-                char_t result[512];
                 char_t* s = result;
 
@@ -5741 +8128 @@
                         while (exponent > 0)
                         {
-                                assert(*mantissa == 0 || (unsigned)(*mantissa - '0') <= 9);
+                                assert(*mantissa == 0 || static_cast<unsigned int>(static_cast<unsigned int>(*mantissa) - '0') <= 9);
                                 *s++ = *mantissa ? *mantissa++ : '0';
                                 exponent--;
@@ -5763 +8150 @@
                         while (*mantissa)
                         {
-                                assert((unsigned)(*mantissa - '0') <= 9);
+                                assert(static_cast<unsigned int>(*mantissa - '0') <= 9);
                                 *s++ = *mantissa++;
                         }
@@ -5769 +8156 @@
 
                 // zero-terminate
-                assert(s < result + sizeof(result) / sizeof(result[0]));
+                assert(s < result + result_size);
                 *s = 0;
 
-                return xpath_string(result, alloc);
-        }
-        
-        bool check_string_to_number_format(const char_t* string)
+                return xpath_string::from_heap_preallocated(result, s);
+        }
+
+        PUGI__FN bool check_string_to_number_format(const char_t* string)
         {
                 // parse leading whitespace
-                while (IS_CHARTYPE(*string, ct_space)) ++string;
+                while (PUGI__IS_CHARTYPE(*string, ct_space)) ++string;
 
                 // parse sign
@@ -5786 +8173 @@
 
                 // if there is no integer part, there should be a decimal part with at least one digit
-                if (!IS_CHARTYPEX(string[0], ctx_digit) && (string[0] != '.' || !IS_CHARTYPEX(string[1], ctx_digit))) return false;
+                if (!PUGI__IS_CHARTYPEX(string[0], ctx_digit) && (string[0] != '.' || !PUGI__IS_CHARTYPEX(string[1], ctx_digit))) return false;
 
                 // parse integer part
-                while (IS_CHARTYPEX(*string, ctx_digit)) ++string;
+                while (PUGI__IS_CHARTYPEX(*string, ctx_digit)) ++string;
 
                 // parse decimal part
@@ -5796 +8183 @@
                         ++string;
 
-                        while (IS_CHARTYPEX(*string, ctx_digit)) ++string;
+                        while (PUGI__IS_CHARTYPEX(*string, ctx_digit)) ++string;
                 }
 
                 // parse trailing whitespace
-                while (IS_CHARTYPE(*string, ct_space)) ++string;
+                while (PUGI__IS_CHARTYPE(*string, ct_space)) ++string;
 
                 return *string == 0;
         }
 
-        double convert_string_to_number(const char_t* string)
+        PUGI__FN double convert_string_to_number(const char_t* string)
         {
                 // check string format
@@ -5814 +8201 @@
                 return wcstod(string, 0);
         #else
-                return atof(string);
+                return strtod(string, 0);
         #endif
         }
 
-        bool convert_string_to_number(const char_t* begin, const char_t* end, double* out_result)
-        {
-                char_t buffer[32];
-
+        PUGI__FN bool convert_string_to_number_scratch(char_t (&buffer)[32], const char_t* begin, const char_t* end, double* out_result)
+        {
                 size_t length = static_cast<size_t>(end - begin);
                 char_t* scratch = buffer;
@@ -5828 +8213 @@
                 {
                         // need to make dummy on-heap copy
-                        scratch = static_cast<char_t*>(global_allocate((length + 1) * sizeof(char_t)));
+                        scratch = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
                         if (!scratch) return false;
                 }
@@ -5839 +8224 @@
 
                 // free dummy buffer
-                if (scratch != buffer) global_deallocate(scratch);
+                if (scratch != buffer) xml_memory::deallocate(scratch);
 
                 return true;
         }
-        
-        double round_nearest(double value)
+
+        PUGI__FN double round_nearest(double value)
         {
                 return floor(value + 0.5);
         }
 
-        double round_nearest_nzero(double value)
+        PUGI__FN double round_nearest_nzero(double value)
         {
                 // same as round_nearest, but returns -0 for [-0.5, -0]
@@ -5855 +8240 @@
                 return (value >= -0.5 && value <= 0) ? ceil(value) : floor(value + 0.5);
         }
-        
-        const char_t* qualified_name(const xpath_node& node)
+
+        PUGI__FN const char_t* qualified_name(const xpath_node& node)
         {
                 return node.attribute() ? node.attribute().name() : node.node().name();
         }
-        
-        const char_t* local_name(const xpath_node& node)
+
+        PUGI__FN const char_t* local_name(const xpath_node& node)
         {
                 const char_t* name = qualified_name(node);
                 const char_t* p = find_char(name, ':');
-                
+
                 return p ? p + 1 : name;
         }
@@ -5882 +8267 @@
                 }
 
-                bool operator()(const xml_attribute& a) const
+                bool operator()(xml_attribute a) const
                 {
                         const char_t* name = a.name();
@@ -5892 +8277 @@
         };
 
-        const char_t* namespace_uri(const xml_node& node)
+        PUGI__FN const char_t* namespace_uri(xml_node node)
         {
                 namespace_uri_predicate pred = node.name();
-                
+
                 xml_node p = node;
-                
+
                 while (p)
                 {
                         xml_attribute a = p.find_attribute(pred);
-                        
+
                         if (a) return a.value();
-                        
+
                         p = p.parent();
                 }
-                
+
                 return PUGIXML_TEXT("");
         }
 
-        const char_t* namespace_uri(const xml_attribute& attr, const xml_node& parent)
+        PUGI__FN const char_t* namespace_uri(xml_attribute attr, xml_node parent)
         {
                 namespace_uri_predicate pred = attr.name();
-                
+
                 // Default namespace does not apply to attributes
                 if (!pred.prefix) return PUGIXML_TEXT("");
-                
+
                 xml_node p = parent;
-                
+
                 while (p)
                 {
                         xml_attribute a = p.find_attribute(pred);
-                        
+
                         if (a) return a.value();
-                        
+
                         p = p.parent();
                 }
-                
+
                 return PUGIXML_TEXT("");
         }
 
-        const char_t* namespace_uri(const xpath_node& node)
+        PUGI__FN const char_t* namespace_uri(const xpath_node& node)
         {
                 return node.attribute() ? namespace_uri(node.attribute(), node.parent()) : namespace_uri(node.node());
         }
 
-        void normalize_space(char_t* buffer)
+        PUGI__FN char_t* normalize_space(char_t* buffer)
         {
                 char_t* write = buffer;
@@ -5944 +8329 @@
                         char_t ch = *it++;
 
-                        if (IS_CHARTYPE(ch, ct_space))
+                        if (PUGI__IS_CHARTYPE(ch, ct_space))
                         {
                                 // replace whitespace sequence with single space
-                                while (IS_CHARTYPE(*it, ct_space)) it++;
+                                while (PUGI__IS_CHARTYPE(*it, ct_space)) it++;
 
                                 // avoid leading spaces
@@ -5956 +8341 @@
 
                 // remove trailing space
-                if (write != buffer && IS_CHARTYPE(write[-1], ct_space)) write--;
+                if (write != buffer && PUGI__IS_CHARTYPE(write[-1], ct_space)) write--;
 
                 // zero-terminate
                 *write = 0;
-        }
-
-        void translate(char_t* buffer, const char_t* from, const char_t* to)
-        {
-                size_t to_length = strlength(to);
-
+
+                return write;
+        }
+
+        PUGI__FN char_t* translate(char_t* buffer, const char_t* from, const char_t* to, size_t to_length)
+        {
                 char_t* write = buffer;
 
                 while (*buffer)
                 {
-                        DMC_VOLATILE char_t ch = *buffer++;
+                        PUGI__DMC_VOLATILE char_t ch = *buffer++;
 
                         const char_t* pos = find_char(from, ch);
@@ -5982 +8367 @@
                 // zero-terminate
                 *write = 0;
+
+                return write;
+        }
+
+        PUGI__FN unsigned char* translate_table_generate(xpath_allocator* alloc, const char_t* from, const char_t* to)
+        {
+                unsigned char table[128] = {0};
+
+                while (*from)
+                {
+                        unsigned int fc = static_cast<unsigned int>(*from);
+                        unsigned int tc = static_cast<unsigned int>(*to);
+
+                        if (fc >= 128 || tc >= 128)
+                                return 0;
+
+                        // code=128 means "skip character"
+                        if (!table[fc])
+                                table[fc] = static_cast<unsigned char>(tc ? tc : 128);
+
+                        from++;
+                        if (tc) to++;
+                }
+
+                for (int i = 0; i < 128; ++i)
+                        if (!table[i])
+                                table[i] = static_cast<unsigned char>(i);
+
+                void* result = alloc->allocate_nothrow(sizeof(table));
+
+                if (result)
+                {
+                        memcpy(result, table, sizeof(table));
+                }
+
+                return static_cast<unsigned char*>(result);
+        }
+
+        PUGI__FN char_t* translate_table(char_t* buffer, const unsigned char* table)
+        {
+                char_t* write = buffer;
+
+                while (*buffer)
+                {
+                        char_t ch = *buffer++;
+                        unsigned int index = static_cast<unsigned int>(ch);
+
+                        if (index < 128)
+                        {
+                                unsigned char code = table[index];
+
+                                // code=128 means "skip character" (table size is 128 so 128 can be a special value)
+                                // this code skips these characters without extra branches
+                                *write = static_cast<char_t>(code);
+                                write += 1 - (code >> 7);
+                        }
+                        else
+                        {
+                                *write++ = ch;
+                        }
+                }
+
+                // zero-terminate
+                *write = 0;
+
+                return write;
+        }
+
+        inline bool is_xpath_attribute(const char_t* name)
+        {
+                return !(starts_with(name, PUGIXML_TEXT("xmlns")) && (name[5] == 0 || name[5] == ':'));
         }
 
         struct xpath_variable_boolean: xpath_variable
         {
-                xpath_variable_boolean(): value(false)
+                xpath_variable_boolean(): xpath_variable(xpath_type_boolean), value(false)
                 {
                 }
@@ -5996 +8452 @@
         struct xpath_variable_number: xpath_variable
         {
-                xpath_variable_number(): value(0)
+                xpath_variable_number(): xpath_variable(xpath_type_number), value(0)
                 {
                 }
@@ -6006 +8462 @@
         struct xpath_variable_string: xpath_variable
         {
-                xpath_variable_string(): value(0)
+                xpath_variable_string(): xpath_variable(xpath_type_string), value(0)
                 {
                 }
@@ -6012 +8468 @@
                 ~xpath_variable_string()
                 {
-                        if (value) global_deallocate(value);
+                        if (value) xml_memory::deallocate(value);
                 }
 
@@ -6021 +8477 @@
         struct xpath_variable_node_set: xpath_variable
         {
+                xpath_variable_node_set(): xpath_variable(xpath_type_node_set)
+                {
+                }
+
                 xpath_node_set value;
                 char_t name[1];
         };
 
-        const xpath_node_set dummy_node_set;
-
-        unsigned int hash_string(const char_t* str)
+        static const xpath_node_set dummy_node_set;
+
+        PUGI__FN unsigned int hash_string(const char_t* str)
         {
                 // Jenkins one-at-a-time hash (http://en.wikipedia.org/wiki/Jenkins_hash_function#one-at-a-time)
@@ -6038 +8498 @@
                         result ^= result >> 6;
                 }
-        
+
                 result += result << 3;
                 result ^= result >> 11;
                 result += result << 15;
-        
+
                 return result;
         }
 
-        template <typename T> T* new_xpath_variable(const char_t* name)
+        template <typename T> PUGI__FN T* new_xpath_variable(const char_t* name)
         {
                 size_t length = strlength(name);
@@ -6052 +8512 @@
 
                 // $$ we can't use offsetof(T, name) because T is non-POD, so we just allocate additional length characters
-                void* memory = global_allocate(sizeof(T) + length * sizeof(char_t));
+                void* memory = xml_memory::allocate(sizeof(T) + length * sizeof(char_t));
                 if (!memory) return 0;
 
@@ -6062 +8522 @@
         }
 
-        xpath_variable* new_xpath_variable(xpath_value_type type, const char_t* name)
+        PUGI__FN xpath_variable* new_xpath_variable(xpath_value_type type, const char_t* name)
         {
                 switch (type)
@@ -6083 +8543 @@
         }
 
-        template <typename T> void delete_xpath_variable(T* var)
+        template <typename T> PUGI__FN void delete_xpath_variable(T* var)
         {
                 var->~T();
-                global_deallocate(var);
-        }
-
-        void delete_xpath_variable(xpath_value_type type, xpath_variable* var)
+                xml_memory::deallocate(var);
+        }
+
+        PUGI__FN void delete_xpath_variable(xpath_value_type type, xpath_variable* var)
         {
                 switch (type)
@@ -6110 +8570 @@
 
                 default:
-                        assert(!"Invalid variable type");
-                }
-        }
-
-        xpath_variable* get_variable(xpath_variable_set* set, const char_t* begin, const char_t* end)
-        {
-                char_t buffer[32];
-
+                        assert(false && "Invalid variable type");
+                }
+        }
+
+        PUGI__FN bool copy_xpath_variable(xpath_variable* lhs, const xpath_variable* rhs)
+        {
+                switch (rhs->type())
+                {
+                case xpath_type_node_set:
+                        return lhs->set(static_cast<const xpath_variable_node_set*>(rhs)->value);
+
+                case xpath_type_number:
+                        return lhs->set(static_cast<const xpath_variable_number*>(rhs)->value);
+
+                case xpath_type_string:
+                        return lhs->set(static_cast<const xpath_variable_string*>(rhs)->value);
+
+                case xpath_type_boolean:
+                        return lhs->set(static_cast<const xpath_variable_boolean*>(rhs)->value);
+
+                default:
+                        assert(false && "Invalid variable type");
+                        return false;
+                }
+        }
+
+        PUGI__FN bool get_variable_scratch(char_t (&buffer)[32], xpath_variable_set* set, const char_t* begin, const char_t* end, xpath_variable** out_result)
+        {
                 size_t length = static_cast<size_t>(end - begin);
                 char_t* scratch = buffer;
@@ -6124 +8604 @@
                 {
                         // need to make dummy on-heap copy
-                        scratch = static_cast<char_t*>(global_allocate((length + 1) * sizeof(char_t)));
-                        if (!scratch) return 0;
+                        scratch = static_cast<char_t*>(xml_memory::allocate((length + 1) * sizeof(char_t)));
+                        if (!scratch) return false;
                 }
 
@@ -6132 +8612 @@
                 scratch[length] = 0;
 
-                xpath_variable* result = set->get(scratch);
+                *out_result = set->get(scratch);
 
                 // free dummy buffer
-                if (scratch != buffer) global_deallocate(scratch);
-
-                return result;
-        }
-}
+                if (scratch != buffer) xml_memory::deallocate(scratch);
+
+                return true;
+        }
+PUGI__NS_END
 
 // Internal node set class
-namespace
-{
-        xpath_node_set::type_t xpath_sort(xpath_node* begin, xpath_node* end, xpath_node_set::type_t type, bool rev)
+PUGI__NS_BEGIN
+        PUGI__FN xpath_node_set::type_t xpath_get_order(const xpath_node* begin, const xpath_node* end)
+        {
+                if (end - begin < 2)
+                        return xpath_node_set::type_sorted;
+
+                document_order_comparator cmp;
+
+                bool first = cmp(begin[0], begin[1]);
+
+                for (const xpath_node* it = begin + 1; it + 1 < end; ++it)
+                        if (cmp(it[0], it[1]) != first)
+                                return xpath_node_set::type_unsorted;
+
+                return first ? xpath_node_set::type_sorted : xpath_node_set::type_sorted_reverse;
+        }
+
+        PUGI__FN xpath_node_set::type_t xpath_sort(xpath_node* begin, xpath_node* end, xpath_node_set::type_t type, bool rev)
         {
                 xpath_node_set::type_t order = rev ? xpath_node_set::type_sorted_reverse : xpath_node_set::type_sorted;
@@ -6150 +8645 @@
                 if (type == xpath_node_set::type_unsorted)
                 {
-                        sort(begin, end, document_order_comparator());
-
-                        type = xpath_node_set::type_sorted;
-                }
-                
+                        xpath_node_set::type_t sorted = xpath_get_order(begin, end);
+
+                        if (sorted == xpath_node_set::type_unsorted)
+                        {
+                                sort(begin, end, document_order_comparator());
+
+                                type = xpath_node_set::type_sorted;
+                        }
+                        else
+                                type = sorted;
+                }
+
                 if (type != order) reverse(begin, end);
-                        
+
                 return order;
         }
 
-        xpath_node xpath_first(const xpath_node* begin, const xpath_node* end, xpath_node_set::type_t type)
+        PUGI__FN xpath_node xpath_first(const xpath_node* begin, const xpath_node* end, xpath_node_set::type_t type)
         {
                 if (begin == end) return xpath_node();
@@ -6176 +8678 @@
 
                 default:
-                        assert(!"Invalid node set type");
+                        assert(false && "Invalid node set type");
                         return xpath_node();
                 }
         }
+
         class xpath_node_set_raw
         {
@@ -6218 +8721 @@
                 }
 
+                void push_back_grow(const xpath_node& node, xpath_allocator* alloc);
+
                 void push_back(const xpath_node& node, xpath_allocator* alloc)
                 {
-                        if (_end == _eos)
-                        {
-                                size_t capacity = static_cast<size_t>(_eos - _begin);
-
-                                // get new capacity (1.5x rule)
-                                size_t new_capacity = capacity + capacity / 2 + 1;
-
+                        if (_end != _eos)
+                                *_end++ = node;
+                        else
+                                push_back_grow(node, alloc);
+                }
+
+                void append(const xpath_node* begin_, const xpath_node* end_, xpath_allocator* alloc)
+                {
+                        if (begin_ == end_) return;
+
+                        size_t size_ = static_cast<size_t>(_end - _begin);
+                        size_t capacity = static_cast<size_t>(_eos - _begin);
+                        size_t count = static_cast<size_t>(end_ - begin_);
+
+                        if (size_ + count > capacity)
+                        {
                                 // reallocate the old array or allocate a new one
-                                xpath_node* data = static_cast<xpath_node*>(alloc->reallocate(_begin, capacity * sizeof(xpath_node), new_capacity * sizeof(xpath_node)));
+                                xpath_node* data = static_cast<xpath_node*>(alloc->reallocate(_begin, capacity * sizeof(xpath_node), (size_ + count) * sizeof(xpath_node)));
                                 assert(data);
 
                                 // finalize
                                 _begin = data;
-                                _end = data + capacity;
-                                _eos = data + new_capacity;
-                        }
-
-                        *_end++ = node;
-                }
-
-                void append(const xpath_node* begin, const xpath_node* end, xpath_allocator* alloc)
-                {
-                        size_t size = static_cast<size_t>(_end - _begin);
-                        size_t capacity = static_cast<size_t>(_eos - _begin);
-                        size_t count = static_cast<size_t>(end - begin);
-
-                        if (size + count > capacity)
-                        {
-                                // reallocate the old array or allocate a new one
-                                xpath_node* data = static_cast<xpath_node*>(alloc->reallocate(_begin, capacity * sizeof(xpath_node), (size + count) * sizeof(xpath_node)));
-                                assert(data);
-
-                                // finalize
-                                _begin = data;
-                                _end = data + size;
-                                _eos = data + size + count;
-                        }
-
-                        memcpy(_end, begin, count * sizeof(xpath_node));
+                                _end = data + size_;
+                                _eos = data + size_ + count;
+                        }
+
+                        memcpy(_end, begin_, count * sizeof(xpath_node));
                         _end += count;
                 }
@@ -6278 +8771 @@
                         if (_type == xpath_node_set::type_unsorted)
                                 sort(_begin, _end, duplicate_comparator());
-                
+
                         _end = unique(_begin, _end);
                 }
@@ -6287 +8780 @@
                 }
 
-                void set_type(xpath_node_set::type_t type)
-                {
-                        _type = type;
+                void set_type(xpath_node_set::type_t value)
+                {
+                        _type = value;
                 }
         };
-}
-
-namespace
-{
+
+        PUGI__FN_NO_INLINE void xpath_node_set_raw::push_back_grow(const xpath_node& node, xpath_allocator* alloc)
+        {
+                size_t capacity = static_cast<size_t>(_eos - _begin);
+
+                // get new capacity (1.5x rule)
+                size_t new_capacity = capacity + capacity / 2 + 1;
+
+                // reallocate the old array or allocate a new one
+                xpath_node* data = static_cast<xpath_node*>(alloc->reallocate(_begin, capacity * sizeof(xpath_node), new_capacity * sizeof(xpath_node)));
+                assert(data);
+
+                // finalize
+                _begin = data;
+                _end = data + capacity;
+                _eos = data + new_capacity;
+
+                // push
+                *_end++ = node;
+        }
+PUGI__NS_END
+
+PUGI__NS_BEGIN
         struct xpath_context
         {
@@ -6301 +8813 @@
                 size_t position, size;
 
-                xpath_context(const xpath_node& n, size_t position, size_t size): n(n), position(position), size(size)
+                xpath_context(const xpath_node& n_, size_t position_, size_t size_): n(n_), position(position_), size(size_)
                 {
                 }
@@ -6367 +8879 @@
                         next();
                 }
-                
+
                 const char_t* state() const
                 {
                         return _cur;
                 }
-                
+
                 void next()
                 {
                         const char_t* cur = _cur;
 
-                        while (IS_CHARTYPE(*cur, ct_space)) ++cur;
+                        while (PUGI__IS_CHARTYPE(*cur, ct_space)) ++cur;
 
                         // save lexeme position for error reporting
@@ -6387 +8899 @@
                                 _cur_lexeme = lex_eof;
                                 break;
-                        
+
                         case '>':
                                 if (*(cur+1) == '=')
@@ -6431 +8943 @@
 
                                 break;
-                        
+
                         case '+':
                                 cur += 1;
@@ -6455 +8967 @@
 
                                 break;
-                        
+
                         case '$':
                                 cur += 1;
 
-                                if (IS_CHARTYPEX(*cur, ctx_start_symbol))
+                                if (PUGI__IS_CHARTYPEX(*cur, ctx_start_symbol))
                                 {
                                         _cur_lexeme_contents.begin = cur;
 
-                                        while (IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
-
-                                        if (cur[0] == ':' && IS_CHARTYPEX(cur[1], ctx_symbol)) // qname
+                                        while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
+
+                                        if (cur[0] == ':' && PUGI__IS_CHARTYPEX(cur[1], ctx_symbol)) // qname
                                         {
                                                 cur++; // :
 
-                                                while (IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
+                                                while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
                                         }
 
                                         _cur_lexeme_contents.end = cur;
-                                
+
                                         _cur_lexeme = lex_var_ref;
                                 }
@@ -6494 +9006 @@
 
                                 break;
-                        
+
                         case '[':
                                 cur += 1;
@@ -6525 +9037 @@
                                 }
                                 break;
-                
+
                         case '.':
                                 if (*(cur+1) == '.')
@@ -6532 +9044 @@
                                         _cur_lexeme = lex_double_dot;
                                 }
-                                else if (IS_CHARTYPEX(*(cur+1), ctx_digit))
+                                else if (PUGI__IS_CHARTYPEX(*(cur+1), ctx_digit))
                                 {
                                         _cur_lexeme_contents.begin = cur; // .
@@ -6538 +9050 @@
                                         ++cur;
 
-                                        while (IS_CHARTYPEX(*cur, ctx_digit)) cur++;
+                                        while (PUGI__IS_CHARTYPEX(*cur, ctx_digit)) cur++;
 
                                         _cur_lexeme_contents.end = cur;
-                                        
+
                                         _cur_lexeme = lex_number;
                                 }
@@ -6567 +9079 @@
                                 while (*cur && *cur != terminator) cur++;
                                 _cur_lexeme_contents.end = cur;
-                                
+
                                 if (!*cur)
                                         _cur_lexeme = lex_none;
@@ -6592 +9104 @@
 
                         default:
-                                if (IS_CHARTYPEX(*cur, ctx_digit))
+                                if (PUGI__IS_CHARTYPEX(*cur, ctx_digit))
                                 {
                                         _cur_lexeme_contents.begin = cur;
 
-                                        while (IS_CHARTYPEX(*cur, ctx_digit)) cur++;
-                                
+                                        while (PUGI__IS_CHARTYPEX(*cur, ctx_digit)) cur++;
+
                                         if (*cur == '.')
                                         {
                                                 cur++;
 
-                                                while (IS_CHARTYPEX(*cur, ctx_digit)) cur++;
+                                                while (PUGI__IS_CHARTYPEX(*cur, ctx_digit)) cur++;
                                         }
 
@@ -6609 +9121 @@
                                         _cur_lexeme = lex_number;
                                 }
-                                else if (IS_CHARTYPEX(*cur, ctx_start_symbol))
+                                else if (PUGI__IS_CHARTYPEX(*cur, ctx_start_symbol))
                                 {
                                         _cur_lexeme_contents.begin = cur;
 
-                                        while (IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
+                                        while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
 
                                         if (cur[0] == ':')
@@ -6621 +9133 @@
                                                         cur += 2; // :*
                                                 }
-                                                else if (IS_CHARTYPEX(cur[1], ctx_symbol)) // namespace test qname
+                                                else if (PUGI__IS_CHARTYPEX(cur[1], ctx_symbol)) // namespace test qname
                                                 {
                                                         cur++; // :
 
-                                                        while (IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
+                                                        while (PUGI__IS_CHARTYPEX(*cur, ctx_symbol)) cur++;
                                                 }
                                         }
 
                                         _cur_lexeme_contents.end = cur;
-                                
+
                                         _cur_lexeme = lex_string;
                                 }
@@ -6662 +9174 @@
         enum ast_type_t
         {
+                ast_unknown,
                 ast_op_or,                                              // left or right
                 ast_op_and,                                             // left and right
                 ast_op_equal,                                   // left = right
-                ast_op_not_equal,                               // left != right
+                ast_op_not_equal,                               // left != right
                 ast_op_less,                                    // left < right
                 ast_op_greater,                                 // left > right
@@ -6679 +9192 @@
                 ast_predicate,                                  // apply predicate to set; next points to next predicate
                 ast_filter,                                             // select * from left where right
-                ast_filter_posinv,                              // select * from left where right; proximity position invariant
                 ast_string_constant,                    // string constant
                 ast_number_constant,                    // number constant
@@ -6719 +9231 @@
                 ast_func_round,                                 // round(left)
                 ast_step,                                               // process set left with step
-                ast_step_root                                   // select root node
+                ast_step_root,                                  // select root node
+
+                ast_opt_translate_table,                // translate(left, right, third) where right/third are constants
+                ast_opt_compare_attribute               // @name = 'string'
         };
 
@@ -6738 +9253 @@
                 axis_self
         };
-        
+
         enum nodetest_t
         {
@@ -6752 +9267 @@
         };
 
+        enum predicate_t
+        {
+                predicate_default,
+                predicate_posinv,
+                predicate_constant,
+                predicate_constant_one
+        };
+
+        enum nodeset_eval_t
+        {
+                nodeset_eval_all,
+                nodeset_eval_any,
+                nodeset_eval_first
+        };
+
         template <axis_t N> struct axis_to_type
         {
@@ -6758 +9288 @@
 
         template <axis_t N> const axis_t axis_to_type<N>::axis = N;
-                
+
         class xpath_ast_node
         {
@@ -6766 +9296 @@
                 char _rettype;
 
-                // for ast_step / ast_predicate
+                // for ast_step
                 char _axis;
+
+                // for ast_step/ast_predicate/ast_filter
                 char _test;
 
@@ -6785 +9317 @@
                         // node test for ast_step (node name/namespace/node type/pi target)
                         const char_t* nodetest;
+                        // table for ast_opt_translate_table
+                        const unsigned char* table;
                 } _data;
 
@@ -6814 +9348 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
-                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
+                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack, nodeset_eval_all);
+                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack, nodeset_eval_all);
 
                                 for (const xpath_node* li = ls.begin(); li != ls.end(); ++li)
@@ -6843 +9377 @@
 
                                         double l = lhs->eval_number(c, stack);
-                                        xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
+                                        xpath_node_set_raw rs = rhs->eval_node_set(c, stack, nodeset_eval_all);
 
                                         for (const xpath_node* ri = rs.begin(); ri != rs.end(); ++ri)
@@ -6860 +9394 @@
 
                                         xpath_string l = lhs->eval_string(c, stack);
-                                        xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
+                                        xpath_node_set_raw rs = rhs->eval_node_set(c, stack, nodeset_eval_all);
 
                                         for (const xpath_node* ri = rs.begin(); ri != rs.end(); ++ri)
@@ -6874 +9408 @@
                         }
 
-                        assert(!"Wrong types");
+                        assert(false && "Wrong types");
                         return false;
+                }
+
+                static bool eval_once(xpath_node_set::type_t type, nodeset_eval_t eval)
+                {
+                        return type == xpath_node_set::type_sorted ? eval != nodeset_eval_all : eval == nodeset_eval_any;
                 }
 
@@ -6888 +9427 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
-                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
+                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack, nodeset_eval_all);
+                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack, nodeset_eval_all);
 
                                 for (const xpath_node* li = ls.begin(); li != ls.end(); ++li)
@@ -6913 +9452 @@
 
                                 double l = lhs->eval_number(c, stack);
-                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack);
+                                xpath_node_set_raw rs = rhs->eval_node_set(c, stack, nodeset_eval_all);
 
                                 for (const xpath_node* ri = rs.begin(); ri != rs.end(); ++ri)
@@ -6929 +9468 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack);
+                                xpath_node_set_raw ls = lhs->eval_node_set(c, stack, nodeset_eval_all);
                                 double r = rhs->eval_number(c, stack);
 
@@ -6944 +9483 @@
                         else
                         {
-                                assert(!"Wrong types");
+                                assert(false && "Wrong types");
                                 return false;
                         }
                 }
 
-                void apply_predicate(xpath_node_set_raw& ns, size_t first, xpath_ast_node* expr, const xpath_stack& stack)
+                static void apply_predicate_boolean(xpath_node_set_raw& ns, size_t first, xpath_ast_node* expr, const xpath_stack& stack, bool once)
                 {
                         assert(ns.size() >= first);
+                        assert(expr->rettype() != xpath_type_number);
 
                         size_t i = 1;
                         size_t size = ns.size() - first;
-                                
+
                         xpath_node* last = ns.begin() + first;
-                                
+
                         // remove_if... or well, sort of
                         for (xpath_node* it = last; it != ns.end(); ++it, ++i)
                         {
                                 xpath_context c(*it, i, size);
-                        
-                                if (expr->rettype() == xpath_type_number)
-                                {
-                                        if (expr->eval_number(c, stack) == i)
-                                                *last++ = *it;
-                                }
-                                else if (expr->eval_boolean(c, stack))
+
+                                if (expr->eval_boolean(c, stack))
+                                {
                                         *last++ = *it;
-                        }
-                        
+
+                                        if (once) break;
+                                }
+                        }
+
                         ns.truncate(last);
                 }
 
-                void apply_predicates(xpath_node_set_raw& ns, size_t first, const xpath_stack& stack)
+                static void apply_predicate_number(xpath_node_set_raw& ns, size_t first, xpath_ast_node* expr, const xpath_stack& stack, bool once)
+                {
+                        assert(ns.size() >= first);
+                        assert(expr->rettype() == xpath_type_number);
+
+                        size_t i = 1;
+                        size_t size = ns.size() - first;
+
+                        xpath_node* last = ns.begin() + first;
+
+                        // remove_if... or well, sort of
+                        for (xpath_node* it = last; it != ns.end(); ++it, ++i)
+                        {
+                                xpath_context c(*it, i, size);
+
+                                if (expr->eval_number(c, stack) == i)
+                                {
+                                        *last++ = *it;
+
+                                        if (once) break;
+                                }
+                        }
+
+                        ns.truncate(last);
+                }
+
+                static void apply_predicate_number_const(xpath_node_set_raw& ns, size_t first, xpath_ast_node* expr, const xpath_stack& stack)
+                {
+                        assert(ns.size() >= first);
+                        assert(expr->rettype() == xpath_type_number);
+
+                        size_t size = ns.size() - first;
+
+                        xpath_node* last = ns.begin() + first;
+
+                        xpath_context c(xpath_node(), 1, size);
+
+                        double er = expr->eval_number(c, stack);
+
+                        if (er >= 1.0 && er <= size)
+                        {
+                                size_t eri = static_cast<size_t>(er);
+
+                                if (er == eri)
+                                {
+                                        xpath_node r = last[eri - 1];
+
+                                        *last++ = r;
+                                }
+                        }
+
+                        ns.truncate(last);
+                }
+
+                void apply_predicate(xpath_node_set_raw& ns, size_t first, const xpath_stack& stack, bool once)
                 {
                         if (ns.size() == first) return;
-                        
+
+                        assert(_type == ast_filter || _type == ast_predicate);
+
+                        if (_test == predicate_constant || _test == predicate_constant_one)
+                                apply_predicate_number_const(ns, first, _right, stack);
+                        else if (_right->rettype() == xpath_type_number)
+                                apply_predicate_number(ns, first, _right, stack, once);
+                        else
+                                apply_predicate_boolean(ns, first, _right, stack, once);
+                }
+
+                void apply_predicates(xpath_node_set_raw& ns, size_t first, const xpath_stack& stack, nodeset_eval_t eval)
+                {
+                        if (ns.size() == first) return;
+
+                        bool last_once = eval_once(ns.type(), eval);
+
                         for (xpath_ast_node* pred = _right; pred; pred = pred->_next)
-                        {
-                                apply_predicate(ns, first, pred->_left, stack);
-                        }
-                }
-
-                void step_push(xpath_node_set_raw& ns, const xml_attribute& a, const xml_node& parent, xpath_allocator* alloc)
-                {
-                        if (!a) return;
-
-                        const char_t* name = a.name();
-
-                        // There are no attribute nodes corresponding to attributes that declare namespaces
-                        // That is, "xmlns:..." or "xmlns"
-                        if (starts_with(name, PUGIXML_TEXT("xmlns")) && (name[5] == 0 || name[5] == ':')) return;
-                        
+                                pred->apply_predicate(ns, first, stack, !pred->_next && last_once);
+                }
+
+                bool step_push(xpath_node_set_raw& ns, xml_attribute_struct* a, xml_node_struct* parent, xpath_allocator* alloc)
+                {
+                        assert(a);
+
+                        const char_t* name = a->name ? a->name + 0 : PUGIXML_TEXT("");
+
                         switch (_test)
                         {
                         case nodetest_name:
-                                if (strequal(name, _data.nodetest)) ns.push_back(xpath_node(a, parent), alloc);
+                                if (strequal(name, _data.nodetest) && is_xpath_attribute(name))
+                                {
+                                        ns.push_back(xpath_node(xml_attribute(a), xml_node(parent)), alloc);
+                                        return true;
+                                }
                                 break;
-                                
+
                         case nodetest_type_node:
                         case nodetest_all:
-                                ns.push_back(xpath_node(a, parent), alloc);
+                                if (is_xpath_attribute(name))
+                                {
+                                        ns.push_back(xpath_node(xml_attribute(a), xml_node(parent)), alloc);
+                                        return true;
+                                }
                                 break;
-                                
+
                         case nodetest_all_in_namespace:
-                                if (starts_with(name, _data.nodetest))
-                                        ns.push_back(xpath_node(a, parent), alloc);
+                                if (starts_with(name, _data.nodetest) && is_xpath_attribute(name))
+                                {
+                                        ns.push_back(xpath_node(xml_attribute(a), xml_node(parent)), alloc);
+                                        return true;
+                                }
                                 break;
-                        
+
                         default:
                                 ;
                         }
-                }
-                
-                void step_push(xpath_node_set_raw& ns, const xml_node& n, xpath_allocator* alloc)
-                {
-                        if (!n) return;
+
+                        return false;
+                }
+
+                bool step_push(xpath_node_set_raw& ns, xml_node_struct* n, xpath_allocator* alloc)
+                {
+                        assert(n);
+
+                        xml_node_type type = PUGI__NODETYPE(n);
 
                         switch (_test)
                         {
                         case nodetest_name:
-                                if (n.type() == node_element && strequal(n.name(), _data.nodetest)) ns.push_back(n, alloc);
+                                if (type == node_element && n->name && strequal(n->name, _data.nodetest))
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
+
                         case nodetest_type_node:
-                                ns.push_back(n, alloc);
+                                ns.push_back(xml_node(n), alloc);
+                                return true;
+
+                        case nodetest_type_comment:
+                                if (type == node_comment)
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
-                        case nodetest_type_comment:
-                                if (n.type() == node_comment)
-                                        ns.push_back(n, alloc);
+
+                        case nodetest_type_text:
+                                if (type == node_pcdata || type == node_cdata)
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
-                        case nodetest_type_text:
-                                if (n.type() == node_pcdata || n.type() == node_cdata)
-                                        ns.push_back(n, alloc);
+
+                        case nodetest_type_pi:
+                                if (type == node_pi)
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
-                        case nodetest_type_pi:
-                                if (n.type() == node_pi)
-                                        ns.push_back(n, alloc);
+
+                        case nodetest_pi:
+                                if (type == node_pi && n->name && strequal(n->name, _data.nodetest))
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                                                        
-                        case nodetest_pi:
-                                if (n.type() == node_pi && strequal(n.name(), _data.nodetest))
-                                        ns.push_back(n, alloc);
+
+                        case nodetest_all:
+                                if (type == node_element)
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
-                        case nodetest_all:
-                                if (n.type() == node_element)
-                                        ns.push_back(n, alloc);
+
+                        case nodetest_all_in_namespace:
+                                if (type == node_element && n->name && starts_with(n->name, _data.nodetest))
+                                {
+                                        ns.push_back(xml_node(n), alloc);
+                                        return true;
+                                }
                                 break;
-                                
-                        case nodetest_all_in_namespace:
-                                if (n.type() == node_element && starts_with(n.name(), _data.nodetest))
-                                        ns.push_back(n, alloc);
+
+                        default:
+                                assert(false && "Unknown axis");
+                        }
+
+                        return false;
+                }
+
+                template <class T> void step_fill(xpath_node_set_raw& ns, xml_node_struct* n, xpath_allocator* alloc, bool once, T)
+                {
+                        const axis_t axis = T::axis;
+
+                        switch (axis)
+                        {
+                        case axis_attribute:
+                        {
+                                for (xml_attribute_struct* a = n->first_attribute; a; a = a->next_attribute)
+                                        if (step_push(ns, a, n, alloc) & once)
+                                                return;
+
                                 break;
-
-                        default:
-                                assert(!"Unknown axis");
-                        } 
-                }
-
-                template <class T> void step_fill(xpath_node_set_raw& ns, const xml_node& n, xpath_allocator* alloc, T)
-                {
-                        const axis_t axis = T::axis;
-
-                        switch (axis)
-                        {
-                        case axis_attribute:
-                        {
-                                for (xml_attribute a = n.first_attribute(); a; a = a.next_attribute())
-                                        step_push(ns, a, n, alloc);
-                                
+                        }
+
+                        case axis_child:
+                        {
+                                for (xml_node_struct* c = n->first_child; c; c = c->next_sibling)
+                                        if (step_push(ns, c, alloc) & once)
+                                                return;
+
                                 break;
                         }
-                        
-                        case axis_child:
-                        {
-                                for (xml_node c = n.first_child(); c; c = c.next_sibling())
-                                        step_push(ns, c, alloc);
-                                        
-                                break;
-                        }
-                        
+
                         case axis_descendant:
                         case axis_descendant_or_self:
                         {
                                 if (axis == axis_descendant_or_self)
-                                        step_push(ns, n, alloc);
-                                        
-                                xml_node cur = n.first_child();
-                                
-                                while (cur && cur != n)
-                                {
-                                        step_push(ns, cur, alloc);
-                                        
-                                        if (cur.first_child())
-                                                cur = cur.first_child();
-                                        else if (cur.next_sibling())
-                                                cur = cur.next_sibling();
+                                        if (step_push(ns, n, alloc) & once)
+                                                return;
+
+                                xml_node_struct* cur = n->first_child;
+
+                                while (cur)
+                                {
+                                        if (step_push(ns, cur, alloc) & once)
+                                                return;
+
+                                        if (cur->first_child)
+                                                cur = cur->first_child;
                                         else
                                         {
-                                                while (!cur.next_sibling() && cur != n)
-                                                        cur = cur.parent();
-                                        
-                                                if (cur != n) cur = cur.next_sibling();
+                                                while (!cur->next_sibling)
+                                                {
+                                                        cur = cur->parent;
+
+                                                        if (cur == n) return;
+                                                }
+
+                                                cur = cur->next_sibling;
                                         }
                                 }
-                                
+
                                 break;
                         }
-                        
+
                         case axis_following_sibling:
                         {
-                                for (xml_node c = n.next_sibling(); c; c = c.next_sibling())
-                                        step_push(ns, c, alloc);
-                                
+                                for (xml_node_struct* c = n->next_sibling; c; c = c->next_sibling)
+                                        if (step_push(ns, c, alloc) & once)
+                                                return;
+
                                 break;
                         }
-                        
+
                         case axis_preceding_sibling:
                         {
-                                for (xml_node c = n.previous_sibling(); c; c = c.previous_sibling())
-                                        step_push(ns, c, alloc);
-                                
+                                for (xml_node_struct* c = n->prev_sibling_c; c->next_sibling; c = c->prev_sibling_c)
+                                        if (step_push(ns, c, alloc) & once)
+                                                return;
+
                                 break;
                         }
-                        
+
                         case axis_following:
                         {
-                                xml_node cur = n;
+                                xml_node_struct* cur = n;
 
                                 // exit from this node so that we don't include descendants
-                                while (cur && !cur.next_sibling()) cur = cur.parent();
-                                cur = cur.next_sibling();
-
-                                for (;;)
-                                {
-                                        step_push(ns, cur, alloc);
-
-                                        if (cur.first_child())
-                                                cur = cur.first_child();
-                                        else if (cur.next_sibling())
-                                                cur = cur.next_sibling();
+                                while (!cur->next_sibling)
+                                {
+                                        cur = cur->parent;
+
+                                        if (!cur) return;
+                                }
+
+                                cur = cur->next_sibling;
+
+                                while (cur)
+                                {
+                                        if (step_push(ns, cur, alloc) & once)
+                                                return;
+
+                                        if (cur->first_child)
+                                                cur = cur->first_child;
                                         else
                                         {
-                                                while (cur && !cur.next_sibling()) cur = cur.parent();
-                                                cur = cur.next_sibling();
-
-                                                if (!cur) break;
+                                                while (!cur->next_sibling)
+                                                {
+                                                        cur = cur->parent;
+
+                                                        if (!cur) return;
+                                                }
+
+                                                cur = cur->next_sibling;
                                         }
                                 }
@@ -7161 +9820 @@
                         case axis_preceding:
                         {
-                                xml_node cur = n;
-
-                                while (cur && !cur.previous_sibling()) cur = cur.parent();
-                                cur = cur.previous_sibling();
-
-                                for (;;)
-                                {
-                                        if (cur.last_child())
-                                                cur = cur.last_child();
+                                xml_node_struct* cur = n;
+
+                                // exit from this node so that we don't include descendants
+                                while (!cur->prev_sibling_c->next_sibling)
+                                {
+                                        cur = cur->parent;
+
+                                        if (!cur) return;
+                                }
+
+                                cur = cur->prev_sibling_c;
+
+                                while (cur)
+                                {
+                                        if (cur->first_child)
+                                                cur = cur->first_child->prev_sibling_c;
                                         else
                                         {
                                                 // leaf node, can't be ancestor
-                                                step_push(ns, cur, alloc);
-
-                                                if (cur.previous_sibling())
-                                                        cur = cur.previous_sibling();
-                                                else
+                                                if (step_push(ns, cur, alloc) & once)
+                                                        return;
+
+                                                while (!cur->prev_sibling_c->next_sibling)
                                                 {
-                                                        do 
-                                                        {
-                                                                cur = cur.parent();
-                                                                if (!cur) break;
-
-                                                                if (!node_is_ancestor(cur, n)) step_push(ns, cur, alloc);
-                                                        }
-                                                        while (!cur.previous_sibling());
-
-                                                        cur = cur.previous_sibling();
-
-                                                        if (!cur) break;
+                                                        cur = cur->parent;
+
+                                                        if (!cur) return;
+
+                                                        if (!node_is_ancestor(cur, n))
+                                                                if (step_push(ns, cur, alloc) & once)
+                                                                        return;
                                                 }
+
+                                                cur = cur->prev_sibling_c;
                                         }
                                 }
@@ -7197 +9859 @@
                                 break;
                         }
-                        
+
                         case axis_ancestor:
                         case axis_ancestor_or_self:
                         {
                                 if (axis == axis_ancestor_or_self)
-                                        step_push(ns, n, alloc);
-
-                                xml_node cur = n.parent();
-                                
+                                        if (step_push(ns, n, alloc) & once)
+                                                return;
+
+                                xml_node_struct* cur = n->parent;
+
                                 while (cur)
                                 {
-                                        step_push(ns, cur, alloc);
-                                        
-                                        cur = cur.parent();
-                                }
-                                
+                                        if (step_push(ns, cur, alloc) & once)
+                                                return;
+
+                                        cur = cur->parent;
+                                }
+
                                 break;
                         }
@@ -7225 +9889 @@
                         case axis_parent:
                         {
-                                if (n.parent()) step_push(ns, n.parent(), alloc);
+                                if (n->parent)
+                                        step_push(ns, n->parent, alloc);
 
                                 break;
                         }
-                                
+
                         default:
-                                assert(!"Unimplemented axis");
-                        }
-                }
-                
-                template <class T> void step_fill(xpath_node_set_raw& ns, const xml_attribute& a, const xml_node& p, xpath_allocator* alloc, T v)
+                                assert(false && "Unimplemented axis");
+                        }
+                }
+
+                template <class T> void step_fill(xpath_node_set_raw& ns, xml_attribute_struct* a, xml_node_struct* p, xpath_allocator* alloc, bool once, T v)
                 {
                         const axis_t axis = T::axis;
@@ -7245 +9910 @@
                         {
                                 if (axis == axis_ancestor_or_self && _test == nodetest_type_node) // reject attributes based on principal node type test
-                                        step_push(ns, a, p, alloc);
-
-                                xml_node cur = p;
-                                
+                                        if (step_push(ns, a, p, alloc) & once)
+                                                return;
+
+                                xml_node_struct* cur = p;
+
                                 while (cur)
                                 {
-                                        step_push(ns, cur, alloc);
-                                        
-                                        cur = cur.parent();
-                                }
-                                
+                                        if (step_push(ns, cur, alloc) & once)
+                                                return;
+
+                                        cur = cur->parent;
+                                }
+
                                 break;
                         }
@@ -7270 +9937 @@
                         case axis_following:
                         {
-                                xml_node cur = p;
-                                
-                                for (;;)
-                                {
-                                        if (cur.first_child())
-                                                cur = cur.first_child();
-                                        else if (cur.next_sibling())
-                                                cur = cur.next_sibling();
+                                xml_node_struct* cur = p;
+
+                                while (cur)
+                                {
+                                        if (cur->first_child)
+                                                cur = cur->first_child;
                                         else
                                         {
-                                                while (cur && !cur.next_sibling()) cur = cur.parent();
-                                                cur = cur.next_sibling();
-                                                
-                                                if (!cur) break;
+                                                while (!cur->next_sibling)
+                                                {
+                                                        cur = cur->parent;
+
+                                                        if (!cur) return;
+                                                }
+
+                                                cur = cur->next_sibling;
                                         }
 
-                                        step_push(ns, cur, alloc);
+                                        if (step_push(ns, cur, alloc) & once)
+                                                return;
                                 }
 
@@ -7302 +9972 @@
                         {
                                 // preceding:: axis does not include attribute nodes and attribute ancestors (they are the same as parent's ancestors), so we can reuse node preceding
-                                step_fill(ns, p, alloc, v);
+                                step_fill(ns, p, alloc, once, v);
                                 break;
                         }
-                        
+
                         default:
-                                assert(!"Unimplemented axis");
-                        }
-                }
-                
-                template <class T> xpath_node_set_raw step_do(const xpath_context& c, const xpath_stack& stack, T v)
+                                assert(false && "Unimplemented axis");
+                        }
+                }
+
+                template <class T> void step_fill(xpath_node_set_raw& ns, const xpath_node& xn, xpath_allocator* alloc, bool once, T v)
                 {
                         const axis_t axis = T::axis;
-                        bool attributes = (axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_descendant_or_self || axis == axis_following || axis == axis_parent || axis == axis_preceding || axis == axis_self);
+                        const bool axis_has_attributes = (axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_descendant_or_self || axis == axis_following || axis == axis_parent || axis == axis_preceding || axis == axis_self);
+
+                        if (xn.node())
+                                step_fill(ns, xn.node().internal_object(), alloc, once, v);
+                        else if (axis_has_attributes && xn.attribute() && xn.parent())
+                                step_fill(ns, xn.attribute().internal_object(), xn.parent().internal_object(), alloc, once, v);
+                }
+
+                template <class T> xpath_node_set_raw step_do(const xpath_context& c, const xpath_stack& stack, nodeset_eval_t eval, T v)
+                {
+                        const axis_t axis = T::axis;
+                        const bool axis_reverse = (axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_preceding || axis == axis_preceding_sibling);
+                        const xpath_node_set::type_t axis_type = axis_reverse ? xpath_node_set::type_sorted_reverse : xpath_node_set::type_sorted;
+
+                        bool once =
+                                (axis == axis_attribute && _test == nodetest_name) ||
+                                (!_right && eval_once(axis_type, eval)) ||
+                                (_right && !_right->_next && _right->_test == predicate_constant_one);
 
                         xpath_node_set_raw ns;
-                        ns.set_type((axis == axis_ancestor || axis == axis_ancestor_or_self || axis == axis_preceding || axis == axis_preceding_sibling) ? xpath_node_set::type_sorted_reverse : xpath_node_set::type_sorted);
+                        ns.set_type(axis_type);
 
                         if (_left)
                         {
-                                xpath_node_set_raw s = _left->eval_node_set(c, stack);
+                                xpath_node_set_raw s = _left->eval_node_set(c, stack, nodeset_eval_all);
 
                                 // self axis preserves the original order
@@ -7332 +10019 @@
                                         // in general, all axes generate elements in a particular order, but there is no order guarantee if axis is applied to two nodes
                                         if (axis != axis_self && size != 0) ns.set_type(xpath_node_set::type_unsorted);
-                                        
-                                        if (it->node())
-                                                step_fill(ns, it->node(), stack.result, v);
-                                        else if (attributes)
-                                                step_fill(ns, it->attribute(), it->parent(), stack.result, v);
-                                                
-                                        apply_predicates(ns, size, stack);
+
+                                        step_fill(ns, *it, stack.result, once, v);
+                                        if (_right) apply_predicates(ns, size, stack, eval);
                                 }
                         }
                         else
                         {
-                                if (c.n.node())
-                                        step_fill(ns, c.n.node(), stack.result, v);
-                                else if (attributes)
-                                        step_fill(ns, c.n.attribute(), c.n.parent(), stack.result, v);
-                                
-                                apply_predicates(ns, 0, stack);
+                                step_fill(ns, c.n, stack.result, once, v);
+                                if (_right) apply_predicates(ns, 0, stack, eval);
                         }
 
@@ -7358 +10037 @@
                         return ns;
                 }
-                
+
         public:
-                xpath_ast_node(ast_type_t type, xpath_value_type rettype, const char_t* value):
-                        _type((char)type), _rettype((char)rettype), _axis(0), _test(0), _left(0), _right(0), _next(0)
+                xpath_ast_node(ast_type_t type, xpath_value_type rettype_, const char_t* value):
+                        _type(static_cast<char>(type)), _rettype(static_cast<char>(rettype_)), _axis(0), _test(0), _left(0), _right(0), _next(0)
                 {
                         assert(type == ast_string_constant);
@@ -7367 +10046 @@
                 }
 
-                xpath_ast_node(ast_type_t type, xpath_value_type rettype, double value):
-                        _type((char)type), _rettype((char)rettype), _axis(0), _test(0), _left(0), _right(0), _next(0)
+                xpath_ast_node(ast_type_t type, xpath_value_type rettype_, double value):
+                        _type(static_cast<char>(type)), _rettype(static_cast<char>(rettype_)), _axis(0), _test(0), _left(0), _right(0), _next(0)
                 {
                         assert(type == ast_number_constant);
                         _data.number = value;
                 }
-                
-                xpath_ast_node(ast_type_t type, xpath_value_type rettype, xpath_variable* value):
-                        _type((char)type), _rettype((char)rettype), _axis(0), _test(0), _left(0), _right(0), _next(0)
+
+                xpath_ast_node(ast_type_t type, xpath_value_type rettype_, xpath_variable* value):
+                        _type(static_cast<char>(type)), _rettype(static_cast<char>(rettype_)), _axis(0), _test(0), _left(0), _right(0), _next(0)
                 {
                         assert(type == ast_variable);
                         _data.variable = value;
                 }
-                
-                xpath_ast_node(ast_type_t type, xpath_value_type rettype, xpath_ast_node* left = 0, xpath_ast_node* right = 0):
-                        _type((char)type), _rettype((char)rettype), _axis(0), _test(0), _left(left), _right(right), _next(0)
+
+                xpath_ast_node(ast_type_t type, xpath_value_type rettype_, xpath_ast_node* left = 0, xpath_ast_node* right = 0):
+                        _type(static_cast<char>(type)), _rettype(static_cast<char>(rettype_)), _axis(0), _test(0), _left(left), _right(right), _next(0)
                 {
                 }
 
                 xpath_ast_node(ast_type_t type, xpath_ast_node* left, axis_t axis, nodetest_t test, const char_t* contents):
-                        _type((char)type), _rettype(xpath_type_node_set), _axis((char)axis), _test((char)test), _left(left), _right(0), _next(0)
-                {
+                        _type(static_cast<char>(type)), _rettype(xpath_type_node_set), _axis(static_cast<char>(axis)), _test(static_cast<char>(test)), _left(left), _right(0), _next(0)
+                {
+                        assert(type == ast_step);
                         _data.nodetest = contents;
+                }
+
+                xpath_ast_node(ast_type_t type, xpath_ast_node* left, xpath_ast_node* right, predicate_t test):
+                        _type(static_cast<char>(type)), _rettype(xpath_type_node_set), _axis(0), _test(static_cast<char>(test)), _left(left), _right(right), _next(0)
+                {
+                        assert(type == ast_filter || type == ast_predicate);
                 }
 
@@ -7408 +10094 @@
                         case ast_op_or:
                                 return _left->eval_boolean(c, stack) || _right->eval_boolean(c, stack);
-                                
+
                         case ast_op_and:
                                 return _left->eval_boolean(c, stack) && _right->eval_boolean(c, stack);
-                                
+
                         case ast_op_equal:
                                 return compare_eq(_left, _right, c, stack, equal_to());
@@ -7417 +10103 @@
                         case ast_op_not_equal:
                                 return compare_eq(_left, _right, c, stack, not_equal_to());
-        
+
                         case ast_op_less:
                                 return compare_rel(_left, _right, c, stack, less());
-                        
+
                         case ast_op_greater:
                                 return compare_rel(_right, _left, c, stack, less());
@@ -7426 +10112 @@
                         case ast_op_less_or_equal:
                                 return compare_rel(_left, _right, c, stack, less_equal());
-                        
+
                         case ast_op_greater_or_equal:
                                 return compare_rel(_right, _left, c, stack, less_equal());
@@ -7452 +10138 @@
                         case ast_func_boolean:
                                 return _left->eval_boolean(c, stack);
-                                
+
                         case ast_func_not:
                                 return !_left->eval_boolean(c, stack);
-                                
+
                         case ast_func_true:
                                 return true;
-                                
+
                         case ast_func_false:
                                 return false;
@@ -7465 +10151 @@
                         {
                                 if (c.n.attribute()) return false;
-                                
+
                                 xpath_allocator_capture cr(stack.result);
 
                                 xpath_string lang = _left->eval_string(c, stack);
-                                
+
                                 for (xml_node n = c.n.node(); n; n = n.parent())
                                 {
                                         xml_attribute a = n.attribute(PUGIXML_TEXT("xml:lang"));
-                                        
+
                                         if (a)
                                         {
                                                 const char_t* value = a.value();
-                                                
+
                                                 // strnicmp / strncasecmp is not portable
                                                 for (const char_t* lit = lang.c_str(); *lit; ++lit)
@@ -7484 +10170 @@
                                                         ++value;
                                                 }
-                                                
+
                                                 return *value == 0 || *value == '-';
                                         }
                                 }
-                                
+
                                 return false;
+                        }
+
+                        case ast_opt_compare_attribute:
+                        {
+                                const char_t* value = (_right->_type == ast_string_constant) ? _right->_data.string : _right->_data.variable->get_string();
+
+                                xml_attribute attr = c.n.node().attribute(_left->_data.nodetest);
+
+                                return attr && strequal(attr.value(), value) && is_xpath_attribute(attr.name());
                         }
 
@@ -7508 +10203 @@
                                 case xpath_type_number:
                                         return convert_number_to_boolean(eval_number(c, stack));
-                                        
+
                                 case xpath_type_string:
                                 {
@@ -7515 +10210 @@
                                         return !eval_string(c, stack).empty();
                                 }
-                                        
-                                case xpath_type_node_set:                               
+
+                                case xpath_type_node_set:
                                 {
                                         xpath_allocator_capture cr(stack.result);
 
-                                        return !eval_node_set(c, stack).empty();
+                                        return !eval_node_set(c, stack, nodeset_eval_any).empty();
                                 }
 
                                 default:
-                                        assert(!"Wrong expression for return type boolean");
+                                        assert(false && "Wrong expression for return type boolean");
                                         return false;
                                 }
@@ -7537 +10232 @@
                         case ast_op_add:
                                 return _left->eval_number(c, stack) + _right->eval_number(c, stack);
-                                
+
                         case ast_op_subtract:
                                 return _left->eval_number(c, stack) - _right->eval_number(c, stack);
@@ -7557 +10252 @@
 
                         case ast_func_last:
-                                return (double)c.size;
-                        
+                                return static_cast<double>(c.size);
+
                         case ast_func_position:
-                                return (double)c.position;
+                                return static_cast<double>(c.position);
 
                         case ast_func_count:
@@ -7566 +10261 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                return (double)_left->eval_node_set(c, stack).size();
-                        }
-                        
+                                return static_cast<double>(_left->eval_node_set(c, stack, nodeset_eval_all).size());
+                        }
+
                         case ast_func_string_length_0:
                         {
                                 xpath_allocator_capture cr(stack.result);
 
-                                return (double)string_value(c.n, stack.result).length();
-                        }
-                        
+                                return static_cast<double>(string_value(c.n, stack.result).length());
+                        }
+
                         case ast_func_string_length_1:
                         {
                                 xpath_allocator_capture cr(stack.result);
 
-                                return (double)_left->eval_string(c, stack).length();
-                        }
-                        
+                                return static_cast<double>(_left->eval_string(c, stack).length());
+                        }
+
                         case ast_func_number_0:
                         {
@@ -7589 +10284 @@
                                 return convert_string_to_number(string_value(c.n, stack.result).c_str());
                         }
-                        
+
                         case ast_func_number_1:
                                 return _left->eval_number(c, stack);
@@ -7598 +10293 @@
 
                                 double r = 0;
-                                
-                                xpath_node_set_raw ns = _left->eval_node_set(c, stack);
-                                
+
+                                xpath_node_set_raw ns = _left->eval_node_set(c, stack, nodeset_eval_all);
+
                                 for (const xpath_node* it = ns.begin(); it != ns.end(); ++it)
                                 {
@@ -7607 +10302 @@
                                         r += convert_string_to_number(string_value(*it, stack.result).c_str());
                                 }
-                        
+
                                 return r;
                         }
@@ -7614 +10309 @@
                         {
                                 double r = _left->eval_number(c, stack);
-                                
+
                                 return r == r ? floor(r) : r;
                         }
@@ -7621 +10316 @@
                         {
                                 double r = _left->eval_number(c, stack);
-                                
+
                                 return r == r ? ceil(r) : r;
                         }
@@ -7627 +10322 @@
                         case ast_func_round:
                                 return round_nearest_nzero(_left->eval_number(c, stack));
-                        
+
                         case ast_variable:
                         {
@@ -7644 +10339 @@
                                 case xpath_type_boolean:
                                         return eval_boolean(c, stack) ? 1 : 0;
-                                        
+
                                 case xpath_type_string:
                                 {
@@ -7651 +10346 @@
                                         return convert_string_to_number(eval_string(c, stack).c_str());
                                 }
-                                        
+
                                 case xpath_type_node_set:
                                 {
@@ -7658 +10353 @@
                                         return convert_string_to_number(eval_string(c, stack).c_str());
                                 }
-                                        
+
                                 default:
-                                        assert(!"Wrong expression for return type number");
+                                        assert(false && "Wrong expression for return type number");
                                         return 0;
                                 }
-                                
-                        }
-                        }
-                }
-                
+
+                        }
+                        }
+                }
+
                 xpath_string eval_string_concat(const xpath_context& c, const xpath_stack& stack)
                 {
@@ -7714 +10409 @@
                         *ri = 0;
 
-                        return xpath_string(result, true);
+                        return xpath_string::from_heap_preallocated(result, ri);
                 }
 
@@ -7722 +10417 @@
                         {
                         case ast_string_constant:
-                                return xpath_string_const(_data.string);
-                        
+                                return xpath_string::from_const(_data.string);
+
                         case ast_func_local_name_0:
                         {
                                 xpath_node na = c.n;
-                                
-                                return xpath_string_const(local_name(na));
+
+                                return xpath_string::from_const(local_name(na));
                         }
 
@@ -7735 +10430 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ns = _left->eval_node_set(c, stack);
+                                xpath_node_set_raw ns = _left->eval_node_set(c, stack, nodeset_eval_first);
                                 xpath_node na = ns.first();
-                                
-                                return xpath_string_const(local_name(na));
+
+                                return xpath_string::from_const(local_name(na));
                         }
 
@@ -7744 +10439 @@
                         {
                                 xpath_node na = c.n;
-                                
-                                return xpath_string_const(qualified_name(na));
+
+                                return xpath_string::from_const(qualified_name(na));
                         }
 
@@ -7752 +10447 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ns = _left->eval_node_set(c, stack);
+                                xpath_node_set_raw ns = _left->eval_node_set(c, stack, nodeset_eval_first);
                                 xpath_node na = ns.first();
-                                
-                                return xpath_string_const(qualified_name(na));
+
+                                return xpath_string::from_const(qualified_name(na));
                         }
 
@@ -7761 +10456 @@
                         {
                                 xpath_node na = c.n;
-                                
-                                return xpath_string_const(namespace_uri(na));
+
+                                return xpath_string::from_const(namespace_uri(na));
                         }
 
@@ -7769 +10464 @@
                                 xpath_allocator_capture cr(stack.result);
 
-                                xpath_node_set_raw ns = _left->eval_node_set(c, stack);
+                                xpath_node_set_raw ns = _left->eval_node_set(c, stack, nodeset_eval_first);
                                 xpath_node na = ns.first();
-                                
-                                return xpath_string_const(namespace_uri(na));
+
+                                return xpath_string::from_const(namespace_uri(na));
                         }
 
@@ -7794 +10489 @@
 
                                 const char_t* pos = find_substring(s.c_str(), p.c_str());
-                                
-                                return pos ? xpath_string(s.c_str(), pos, stack.result) : xpath_string();
-                        }
-                        
+
+                                return pos ? xpath_string::from_heap(s.c_str(), pos, stack.result) : xpath_string();
+                        }
+
                         case ast_func_substring_after:
                         {
@@ -7806 +10501 @@
                                 xpath_string s = _left->eval_string(c, swapped_stack);
                                 xpath_string p = _right->eval_string(c, swapped_stack);
-                                
+
                                 const char_t* pos = find_substring(s.c_str(), p.c_str());
                                 if (!pos) return xpath_string();
 
-                                const char_t* result = pos + p.length();
-
-                                return s.uses_heap() ? xpath_string(result, stack.result) : xpath_string_const(result);
+                                const char_t* rbegin = pos + p.length();
+                                const char_t* rend = s.c_str() + s.length();
+
+                                return s.uses_heap() ? xpath_string::from_heap(rbegin, rend, stack.result) : xpath_string::from_const(rbegin);
                         }
 
@@ -7825 +10521 @@
 
                                 double first = round_nearest(_right->eval_number(c, stack));
-                                
+
                                 if (is_nan(first)) return xpath_string(); // NaN
                                 else if (first >= s_length + 1) return xpath_string();
-                                
-                                size_t pos = first < 1 ? 1 : (size_t)first;
+
+                                size_t pos = first < 1 ? 1 : static_cast<size_t>(first);
                                 assert(1 <= pos && pos <= s_length + 1);
 
                                 const char_t* rbegin = s.c_str() + (pos - 1);
-                                
-                                return s.uses_heap() ? xpath_string(rbegin, stack.result) : xpath_string_const(rbegin);
-                        }
-                        
+                                const char_t* rend = s.c_str() + s.length();
+
+                                return s.uses_heap() ? xpath_string::from_heap(rbegin, rend, stack.result) : xpath_string::from_const(rbegin);
+                        }
+
                         case ast_func_substring_3:
                         {
@@ -7848 +10545 @@
                                 double first = round_nearest(_right->eval_number(c, stack));
                                 double last = first + round_nearest(_right->_next->eval_number(c, stack));
-                                
+
                                 if (is_nan(first) || is_nan(last)) return xpath_string();
                                 else if (first >= s_length + 1) return xpath_string();
                                 else if (first >= last) return xpath_string();
                                 else if (last < 1) return xpath_string();
-                                
-                                size_t pos = first < 1 ? 1 : (size_t)first;
-                                size_t end = last >= s_length + 1 ? s_length + 1 : (size_t)last;
+
+                                size_t pos = first < 1 ? 1 : static_cast<size_t>(first);
+                                size_t end = last >= s_length + 1 ? s_length + 1 : static_cast<size_t>(last);
 
                                 assert(1 <= pos && pos <= end && end <= s_length + 1);
@@ -7861 +10558 @@
                                 const char_t* rend = s.c_str() + (end - 1);
 
-                                return (end == s_length + 1 && !s.uses_heap()) ? xpath_string_const(rbegin) : xpath_string(rbegin, rend, stack.result);
+                                return (end == s_length + 1 && !s.uses_heap()) ? xpath_string::from_const(rbegin) : xpath_string::from_heap(rbegin, rend, stack.result);
                         }
 
@@ -7868 +10565 @@
                                 xpath_string s = string_value(c.n, stack.result);
 
-                                normalize_space(s.data(stack.result));
-
-                                return s;
+                                char_t* begin = s.data(stack.result);
+                                char_t* end = normalize_space(begin);
+
+                                return xpath_string::from_heap_preallocated(begin, end);
                         }
 
@@ -7877 +10575 @@
                                 xpath_string s = _left->eval_string(c, stack);
 
-                                normalize_space(s.data(stack.result));
-                        
-                                return s;
+                                char_t* begin = s.data(stack.result);
+                                char_t* end = normalize_space(begin);
+
+                                return xpath_string::from_heap_preallocated(begin, end);
                         }
 
@@ -7892 +10591 @@
                                 xpath_string to = _right->_next->eval_string(c, swapped_stack);
 
-                                translate(s.data(stack.result), from.c_str(), to.c_str());
-
-                                return s;
+                                char_t* begin = s.data(stack.result);
+                                char_t* end = translate(begin, from.c_str(), to.c_str(), to.length());
+
+                                return xpath_string::from_heap_preallocated(begin, end);
+                        }
+
+                        case ast_opt_translate_table:
+                        {
+                                xpath_string s = _left->eval_string(c, stack);
+
+                                char_t* begin = s.data(stack.result);
+                                char_t* end = translate_table(begin, _data.table);
+
+                                return xpath_string::from_heap_preallocated(begin, end);
                         }
 
@@ -7902 +10612 @@
 
                                 if (_rettype == xpath_type_string)
-                                        return xpath_string_const(_data.variable->get_string());
+                                        return xpath_string::from_const(_data.variable->get_string());
 
                                 // fallthrough to type conversion
@@ -7912 +10622 @@
                                 {
                                 case xpath_type_boolean:
-                                        return xpath_string_const(eval_boolean(c, stack) ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"));
-                                        
+                                        return xpath_string::from_const(eval_boolean(c, stack) ? PUGIXML_TEXT("true") : PUGIXML_TEXT("false"));
+
                                 case xpath_type_number:
                                         return convert_number_to_string(eval_number(c, stack), stack.result);
-                                        
+
                                 case xpath_type_node_set:
                                 {
@@ -7923 +10633 @@
                                         xpath_stack swapped_stack = {stack.temp, stack.result};
 
-                                        xpath_node_set_raw ns = eval_node_set(c, swapped_stack);
+                                        xpath_node_set_raw ns = eval_node_set(c, swapped_stack, nodeset_eval_first);
                                         return ns.empty() ? xpath_string() : string_value(ns.first(), stack.result);
                                 }
-                                
+
                                 default:
-                                        assert(!"Wrong expression for return type string");
+                                        assert(false && "Wrong expression for return type string");
                                         return xpath_string();
                                 }
@@ -7935 +10645 @@
                 }
 
-                xpath_node_set_raw eval_node_set(const xpath_context& c, const xpath_stack& stack)
+                xpath_node_set_raw eval_node_set(const xpath_context& c, const xpath_stack& stack, nodeset_eval_t eval)
                 {
                         switch (_type)
@@ -7945 +10655 @@
                                 xpath_stack swapped_stack = {stack.temp, stack.result};
 
-                                xpath_node_set_raw ls = _left->eval_node_set(c, swapped_stack);
-                                xpath_node_set_raw rs = _right->eval_node_set(c, stack);
-                                
+                                xpath_node_set_raw ls = _left->eval_node_set(c, swapped_stack, eval);
+                                xpath_node_set_raw rs = _right->eval_node_set(c, stack, eval);
+
                                 // we can optimize merging two sorted sets, but this is a very rare operation, so don't bother
-                        rs.set_type(xpath_node_set::type_unsorted);
+                                rs.set_type(xpath_node_set::type_unsorted);
 
                                 rs.append(ls.begin(), ls.end(), stack.result);
                                 rs.remove_duplicates();
-                                
+
                                 return rs;
                         }
 
                         case ast_filter:
-                        case ast_filter_posinv:
-                        {
-                                xpath_node_set_raw set = _left->eval_node_set(c, stack);
+                        {
+                                xpath_node_set_raw set = _left->eval_node_set(c, stack, _test == predicate_constant_one ? nodeset_eval_first : nodeset_eval_all);
 
                                 // either expression is a number or it contains position() call; sort by document order
-                                if (_type == ast_filter) set.sort_do();
-
-                                apply_predicate(set, 0, _right, stack);
-                        
+                                if (_test != predicate_posinv) set.sort_do();
+
+                                bool once = eval_once(set.type(), eval);
+
+                                apply_predicate(set, 0, stack, once);
+
                                 return set;
                         }
-                        
+
                         case ast_func_id:
                                 return xpath_node_set_raw();
-                        
+
                         case ast_step:
                         {
@@ -7978 +10689 @@
                                 {
                                 case axis_ancestor:
-                                        return step_do(c, stack, axis_to_type<axis_ancestor>());
-                                        
+                                        return step_do(c, stack, eval, axis_to_type<axis_ancestor>());
+
                                 case axis_ancestor_or_self:
-                                        return step_do(c, stack, axis_to_type<axis_ancestor_or_self>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_ancestor_or_self>());
 
                                 case axis_attribute:
-                                        return step_do(c, stack, axis_to_type<axis_attribute>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_attribute>());
 
                                 case axis_child:
-                                        return step_do(c, stack, axis_to_type<axis_child>());
-                                
+                                        return step_do(c, stack, eval, axis_to_type<axis_child>());
+
                                 case axis_descendant:
-                                        return step_do(c, stack, axis_to_type<axis_descendant>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_descendant>());
 
                                 case axis_descendant_or_self:
-                                        return step_do(c, stack, axis_to_type<axis_descendant_or_self>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_descendant_or_self>());
 
                                 case axis_following:
-                                        return step_do(c, stack, axis_to_type<axis_following>());
-                                
+                                        return step_do(c, stack, eval, axis_to_type<axis_following>());
+
                                 case axis_following_sibling:
-                                        return step_do(c, stack, axis_to_type<axis_following_sibling>());
-                                
+                                        return step_do(c, stack, eval, axis_to_type<axis_following_sibling>());
+
                                 case axis_namespace:
                                         // namespaced axis is not supported
                                         return xpath_node_set_raw();
-                                
+
                                 case axis_parent:
-                                        return step_do(c, stack, axis_to_type<axis_parent>());
-                                
+                                        return step_do(c, stack, eval, axis_to_type<axis_parent>());
+
                                 case axis_preceding:
-                                        return step_do(c, stack, axis_to_type<axis_preceding>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_preceding>());
 
                                 case axis_preceding_sibling:
-                                        return step_do(c, stack, axis_to_type<axis_preceding_sibling>());
-                                
+                                        return step_do(c, stack, eval, axis_to_type<axis_preceding_sibling>());
+
                                 case axis_self:
-                                        return step_do(c, stack, axis_to_type<axis_self>());
+                                        return step_do(c, stack, eval, axis_to_type<axis_self>());
+
+                                default:
+                                        assert(false && "Unknown axis");
+                                        return xpath_node_set_raw();
                                 }
                         }
@@ -8053 +10768 @@
 
                         default:
-                                assert(!"Wrong expression for return type node set");
+                                assert(false && "Wrong expression for return type node set");
                                 return xpath_node_set_raw();
                         }
                 }
-                
-                bool is_posinv()
+
+                void optimize(xpath_allocator* alloc)
+                {
+                        if (_left)
+                                _left->optimize(alloc);
+
+                        if (_right)
+                                _right->optimize(alloc);
+
+                        if (_next)
+                                _next->optimize(alloc);
+
+                        optimize_self(alloc);
+                }
+
+                void optimize_self(xpath_allocator* alloc)
+                {
+                        // Rewrite [position()=expr] with [expr]
+                        // Note that this step has to go before classification to recognize [position()=1]
+                        if ((_type == ast_filter || _type == ast_predicate) &&
+                                _right->_type == ast_op_equal && _right->_left->_type == ast_func_position && _right->_right->_rettype == xpath_type_number)
+                        {
+                                _right = _right->_right;
+                        }
+
+                        // Classify filter/predicate ops to perform various optimizations during evaluation
+                        if (_type == ast_filter || _type == ast_predicate)
+                        {
+                                assert(_test == predicate_default);
+
+                                if (_right->_type == ast_number_constant && _right->_data.number == 1.0)
+                                        _test = predicate_constant_one;
+                                else if (_right->_rettype == xpath_type_number && (_right->_type == ast_number_constant || _right->_type == ast_variable || _right->_type == ast_func_last))
+                                        _test = predicate_constant;
+                                else if (_right->_rettype != xpath_type_number && _right->is_posinv_expr())
+                                        _test = predicate_posinv;
+                        }
+
+                        // Rewrite descendant-or-self::node()/child::foo with descendant::foo
+                        // The former is a full form of //foo, the latter is much faster since it executes the node test immediately
+                        // Do a similar kind of rewrite for self/descendant/descendant-or-self axes
+                        // Note that we only rewrite positionally invariant steps (//foo[1] != /descendant::foo[1])
+                        if (_type == ast_step && (_axis == axis_child || _axis == axis_self || _axis == axis_descendant || _axis == axis_descendant_or_self) && _left &&
+                                _left->_type == ast_step && _left->_axis == axis_descendant_or_self && _left->_test == nodetest_type_node && !_left->_right &&
+                                is_posinv_step())
+                        {
+                                if (_axis == axis_child || _axis == axis_descendant)
+                                        _axis = axis_descendant;
+                                else
+                                        _axis = axis_descendant_or_self;
+
+                                _left = _left->_left;
+                        }
+
+                        // Use optimized lookup table implementation for translate() with constant arguments
+                        if (_type == ast_func_translate && _right->_type == ast_string_constant && _right->_next->_type == ast_string_constant)
+                        {
+                                unsigned char* table = translate_table_generate(alloc, _right->_data.string, _right->_next->_data.string);
+
+                                if (table)
+                                {
+                                        _type = ast_opt_translate_table;
+                                        _data.table = table;
+                                }
+                        }
+
+                        // Use optimized path for @attr = 'value' or @attr = $value
+                        if (_type == ast_op_equal &&
+                                _left->_type == ast_step && _left->_axis == axis_attribute && _left->_test == nodetest_name && !_left->_left && !_left->_right &&
+                                (_right->_type == ast_string_constant || (_right->_type == ast_variable && _right->_rettype == xpath_type_string)))
+                        {
+                                _type = ast_opt_compare_attribute;
+                        }
+                }
+
+                bool is_posinv_expr() const
                 {
                         switch (_type)
                         {
                         case ast_func_position:
+                        case ast_func_last:
                                 return false;
 
@@ -8076 +10866 @@
                         case ast_predicate:
                         case ast_filter:
-                        case ast_filter_posinv:
                                 return true;
 
                         default:
-                                if (_left && !_left->is_posinv()) return false;
-                                
+                                if (_left && !_left->is_posinv_expr()) return false;
+
                                 for (xpath_ast_node* n = _right; n; n = n->_next)
-                                        if (!n->is_posinv()) return false;
-                                        
+                                        if (!n->is_posinv_expr()) return false;
+
                                 return true;
                         }
                 }
 
+                bool is_posinv_step() const
+                {
+                        assert(_type == ast_step);
+
+                        for (xpath_ast_node* n = _right; n; n = n->_next)
+                        {
+                                assert(n->_type == ast_predicate);
+
+                                if (n->_test != predicate_posinv)
+                                        return false;
+                        }
+
+                        return true;
+                }
+
                 xpath_value_type rettype() const
                 {
@@ -8097 +10901 @@
         struct xpath_parser
         {
-            xpath_allocator* _alloc;
-            xpath_lexer _lexer;
+                xpath_allocator* _alloc;
+                xpath_lexer _lexer;
 
                 const char_t* _query;
@@ -8104 +10908 @@
 
                 xpath_parse_result* _result;
+
+                char_t _scratch[32];
 
         #ifdef PUGIXML_NO_EXCEPTIONS
@@ -8122 +10928 @@
 
                 void throw_error_oom()
-        {
-        #ifdef PUGIXML_NO_EXCEPTIONS
-            throw_error("Out of memory");
-        #else
-            throw std::bad_alloc();
-        #endif
-        }
+                {
+                #ifdef PUGIXML_NO_EXCEPTIONS
+                        throw_error("Out of memory");
+                #else
+                        throw std::bad_alloc();
+                #endif
+                }
 
                 void* alloc_node()
@@ -8147 +10953 @@
                                 char_t* c = static_cast<char_t*>(_alloc->allocate_nothrow((length + 1) * sizeof(char_t)));
                                 if (!c) throw_error_oom();
+                                assert(c); // workaround for clang static analysis
 
                                 memcpy(c, value.begin, length * sizeof(char_t));
@@ -8160 +10967 @@
                         assert(argc <= 1);
 
-                        if (argc == 1 && args[0]->rettype() != xpath_type_node_set) throw_error("Function has to be applied to node set");
+                        if (argc == 1 && args[0]->rettype() != xpath_type_node_set)
+                                throw_error("Function has to be applied to node set");
 
                         return new (alloc_node()) xpath_ast_node(argc == 0 ? type0 : type1, xpath_type_string, args[0]);
@@ -8172 +10980 @@
                                 if (name == PUGIXML_TEXT("boolean") && argc == 1)
                                         return new (alloc_node()) xpath_ast_node(ast_func_boolean, xpath_type_boolean, args[0]);
-                                        
+
                                 break;
-                        
+
                         case 'c':
                                 if (name == PUGIXML_TEXT("count") && argc == 1)
                                 {
-                                        if (args[0]->rettype() != xpath_type_node_set) throw_error("Function has to be applied to node set");
+                                        if (args[0]->rettype() != xpath_type_node_set)
+                                                throw_error("Function has to be applied to node set");
+
                                         return new (alloc_node()) xpath_ast_node(ast_func_count, xpath_type_number, args[0]);
                                 }
                                 else if (name == PUGIXML_TEXT("contains") && argc == 2)
-                                        return new (alloc_node()) xpath_ast_node(ast_func_contains, xpath_type_string, args[0], args[1]);
+                                        return new (alloc_node()) xpath_ast_node(ast_func_contains, xpath_type_boolean, args[0], args[1]);
                                 else if (name == PUGIXML_TEXT("concat") && argc >= 2)
                                         return new (alloc_node()) xpath_ast_node(ast_func_concat, xpath_type_string, args[0], args[1]);
                                 else if (name == PUGIXML_TEXT("ceiling") && argc == 1)
                                         return new (alloc_node()) xpath_ast_node(ast_func_ceiling, xpath_type_number, args[0]);
-                                        
+
                                 break;
-                        
+
                         case 'f':
                                 if (name == PUGIXML_TEXT("false") && argc == 0)
@@ -8195 +11005 @@
                                 else if (name == PUGIXML_TEXT("floor") && argc == 1)
                                         return new (alloc_node()) xpath_ast_node(ast_func_floor, xpath_type_number, args[0]);
-                                        
+
                                 break;
-                        
+
                         case 'i':
                                 if (name == PUGIXML_TEXT("id") && argc == 1)
                                         return new (alloc_node()) xpath_ast_node(ast_func_id, xpath_type_node_set, args[0]);
-                                        
+
                                 break;
-                        
+
                         case 'l':
                                 if (name == PUGIXML_TEXT("last") && argc == 0)
@@ -8211 +11021 @@
                                 else if (name == PUGIXML_TEXT("local-name") && argc <= 1)
                                         return parse_function_helper(ast_func_local_name_0, ast_func_local_name_1, argc, args);
-                        
+
                                 break;
-                        
+
                         case 'n':
                                 if (name == PUGIXML_TEXT("name") && argc <= 1)
@@ -8225 +11035 @@
                                 else if (name == PUGIXML_TEXT("number") && argc <= 1)
                                         return new (alloc_node()) xpath_ast_node(argc == 0 ? ast_func_number_0 : ast_func_number_1, xpath_type_number, args[0]);
-                        
+
                                 break;
-                        
+
                         case 'p':
                                 if (name == PUGIXML_TEXT("position") && argc == 0)
                                         return new (alloc_node()) xpath_ast_node(ast_func_position, xpath_type_number);
-                                
+
                                 break;
-                        
+
                         case 'r':
                                 if (name == PUGIXML_TEXT("round") && argc == 1)
@@ -8239 +11049 @@
 
                                 break;
-                        
+
                         case 's':
                                 if (name == PUGIXML_TEXT("string") && argc <= 1)
                                         return new (alloc_node()) xpath_ast_node(argc == 0 ? ast_func_string_0 : ast_func_string_1, xpath_type_string, args[0]);
                                 else if (name == PUGIXML_TEXT("string-length") && argc <= 1)
-                                        return new (alloc_node()) xpath_ast_node(argc == 0 ? ast_func_string_length_0 : ast_func_string_length_1, xpath_type_string, args[0]);
+                                        return new (alloc_node()) xpath_ast_node(argc == 0 ? ast_func_string_length_0 : ast_func_string_length_1, xpath_type_number, args[0]);
                                 else if (name == PUGIXML_TEXT("starts-with") && argc == 2)
                                         return new (alloc_node()) xpath_ast_node(ast_func_starts_with, xpath_type_boolean, args[0], args[1]);
@@ -8260 +11070 @@
 
                                 break;
-                        
+
                         case 't':
                                 if (name == PUGIXML_TEXT("translate") && argc == 3)
@@ -8266 +11076 @@
                                 else if (name == PUGIXML_TEXT("true") && argc == 0)
                                         return new (alloc_node()) xpath_ast_node(ast_func_true, xpath_type_boolean);
-                                        
+
+                                break;
+
+                        default:
                                 break;
                         }
@@ -8288 +11101 @@
                                 else if (name == PUGIXML_TEXT("attribute"))
                                         return axis_attribute;
-                                
+
                                 break;
-                        
+
                         case 'c':
                                 if (name == PUGIXML_TEXT("child"))
                                         return axis_child;
-                                
+
                                 break;
-                        
+
                         case 'd':
                                 if (name == PUGIXML_TEXT("descendant"))
@@ -8302 +11115 @@
                                 else if (name == PUGIXML_TEXT("descendant-or-self"))
                                         return axis_descendant_or_self;
-                                
+
                                 break;
-                        
+
                         case 'f':
                                 if (name == PUGIXML_TEXT("following"))
@@ -8310 +11123 @@
                                 else if (name == PUGIXML_TEXT("following-sibling"))
                                         return axis_following_sibling;
-                                
+
                                 break;
-                        
+
                         case 'n':
                                 if (name == PUGIXML_TEXT("namespace"))
                                         return axis_namespace;
-                                
+
                                 break;
-                        
+
                         case 'p':
                                 if (name == PUGIXML_TEXT("parent"))
@@ -8326 +11139 @@
                                 else if (name == PUGIXML_TEXT("preceding-sibling"))
                                         return axis_preceding_sibling;
-                                
+
                                 break;
-                        
+
                         case 's':
                                 if (name == PUGIXML_TEXT("self"))
                                         return axis_self;
-                                
+
+                                break;
+
+                        default:
                                 break;
                         }
@@ -8367 +11183 @@
 
                                 break;
+
+                        default:
+                                break;
                         }
 
@@ -8372 +11191 @@
                 }
 
-            // PrimaryExpr ::= VariableReference | '(' Expr ')' | Literal | Number | FunctionCall
-            xpath_ast_node* parse_primary_expression()
-            {
-                switch (_lexer.current())
-                {
-                case lex_var_ref:
-                {
+                // PrimaryExpr ::= VariableReference | '(' Expr ')' | Literal | Number | FunctionCall
+                xpath_ast_node* parse_primary_expression()
+                {
+                        switch (_lexer.current())
+                        {
+                        case lex_var_ref:
+                        {
                                 xpath_lexer_string name = _lexer.contents();
 
@@ -8384 +11203 @@
                                         throw_error("Unknown variable: variable set is not provided");
 
-                                xpath_variable* var = get_variable(_variables, name.begin, name.end);
+                                xpath_variable* var = 0;
+                                if (!get_variable_scratch(_scratch, _variables, name.begin, name.end, &var))
+                                        throw_error_oom();
 
                                 if (!var)
@@ -8391 +11212 @@
                                 _lexer.next();
 
-                        return new (alloc_node()) xpath_ast_node(ast_variable, var->type(), var);
+                                return new (alloc_node()) xpath_ast_node(ast_variable, var->type(), var);
                         }
 
@@ -8422 +11243 @@
                                 double value = 0;
 
-                                if (!convert_string_to_number(_lexer.contents().begin, _lexer.contents().end, &value))
+                                if (!convert_string_to_number_scratch(_scratch, _lexer.contents().begin, _lexer.contents().end, &value))
                                         throw_error_oom();
 
@@ -8435 +11256 @@
                                 xpath_ast_node* args[2] = {0};
                                 size_t argc = 0;
-                                
+
                                 xpath_lexer_string function = _lexer.contents();
                                 _lexer.next();
-                                
+
                                 xpath_ast_node* last_arg = 0;
-                                
+
                                 if (_lexer.current() != lex_open_brace)
                                         throw_error("Unrecognized function call");
@@ -8453 +11274 @@
                                                 throw_error("No comma between function arguments");
                                         _lexer.next();
-                                        
+
                                         xpath_ast_node* n = parse_expression();
-                                        
+
                                         if (argc < 2) args[argc] = n;
                                         else last_arg->set_next(n);
@@ -8462 +11283 @@
                                         last_arg = n;
                                 }
-                                
+
                                 _lexer.next();
 
@@ -8468 +11289 @@
                         }
 
-                default:
-                        throw_error("Unrecognizable primary expression");
-
-                        return 0;
-                }
-            }
-            
-            // FilterExpr ::= PrimaryExpr | FilterExpr Predicate
-            // Predicate ::= '[' PredicateExpr ']'
-            // PredicateExpr ::= Expr
-            xpath_ast_node* parse_filter_expression()
-            {
-                xpath_ast_node* n = parse_primary_expression();
-
-                while (_lexer.current() == lex_open_square_brace)
-                {
-                        _lexer.next();
+                        default:
+                                throw_error("Unrecognizable primary expression");
+
+                                return 0;
+                        }
+                }
+
+                // FilterExpr ::= PrimaryExpr | FilterExpr Predicate
+                // Predicate ::= '[' PredicateExpr ']'
+                // PredicateExpr ::= Expr
+                xpath_ast_node* parse_filter_expression()
+                {
+                        xpath_ast_node* n = parse_primary_expression();
+
+                        while (_lexer.current() == lex_open_square_brace)
+                        {
+                                _lexer.next();
 
                                 xpath_ast_node* expr = parse_expression();
 
-                                if (n->rettype() != xpath_type_node_set) throw_error("Predicate has to be applied to node set");
-
-                                bool posinv = expr->rettype() != xpath_type_number && expr->is_posinv();
-
-                        n = new (alloc_node()) xpath_ast_node(posinv ? ast_filter_posinv : ast_filter, xpath_type_node_set, n, expr);
-
-                        if (_lexer.current() != lex_close_square_brace)
-                                throw_error("Unmatched square brace");
-                
-                        _lexer.next();
-                }
-                
-                return n;
-            }
-            
-            // Step ::= AxisSpecifier NodeTest Predicate* | AbbreviatedStep
-            // AxisSpecifier ::= AxisName '::' | '@'?
-            // NodeTest ::= NameTest | NodeType '(' ')' | 'processing-instruction' '(' Literal ')'
-            // NameTest ::= '*' | NCName ':' '*' | QName
-            // AbbreviatedStep ::= '.' | '..'
-            xpath_ast_node* parse_step(xpath_ast_node* set)
-            {
+                                if (n->rettype() != xpath_type_node_set)
+                                        throw_error("Predicate has to be applied to node set");
+
+                                n = new (alloc_node()) xpath_ast_node(ast_filter, n, expr, predicate_default);
+
+                                if (_lexer.current() != lex_close_square_brace)
+                                        throw_error("Unmatched square brace");
+
+                                _lexer.next();
+                        }
+
+                        return n;
+                }
+
+                // Step ::= AxisSpecifier NodeTest Predicate* | AbbreviatedStep
+                // AxisSpecifier ::= AxisName '::' | '@'?
+                // NodeTest ::= NameTest | NodeType '(' ')' | 'processing-instruction' '(' Literal ')'
+                // NameTest ::= '*' | NCName ':' '*' | QName
+                // AbbreviatedStep ::= '.' | '..'
+                xpath_ast_node* parse_step(xpath_ast_node* set)
+                {
                         if (set && set->rettype() != xpath_type_node_set)
                                 throw_error("Step has to be applied to node set");
@@ -8520 +11340 @@
                                 axis = axis_attribute;
                                 axis_specified = true;
-                                
+
                                 _lexer.next();
                         }
@@ -8526 +11346 @@
                         {
                                 _lexer.next();
-                                
+
                                 return new (alloc_node()) xpath_ast_node(ast_step, set, axis_self, nodetest_type_node, 0);
                         }
@@ -8532 +11352 @@
                         {
                                 _lexer.next();
-                                
+
                                 return new (alloc_node()) xpath_ast_node(ast_step, set, axis_parent, nodetest_type_node, 0);
                         }
-            
+
                         nodetest_t nt_type = nodetest_none;
                         xpath_lexer_string nt_name;
-                        
+
                         if (_lexer.current() == lex_string)
                         {
@@ -8549 +11369 @@
                                 {
                                         // parse axis name
-                                        if (axis_specified) throw_error("Two axis specifiers in one step");
+                                        if (axis_specified)
+                                                throw_error("Two axis specifiers in one step");
 
                                         axis = parse_axis_name(nt_name, axis_specified);
 
-                                        if (!axis_specified) throw_error("Unknown axis");
+                                        if (!axis_specified)
+                                                throw_error("Unknown axis");
 
                                         // read actual node test
@@ -8571 +11393 @@
                                         else throw_error("Unrecognized node test");
                                 }
-                                
+
                                 if (nt_type == nodetest_none)
                                 {
@@ -8578 +11400 @@
                                         {
                                                 _lexer.next();
-                                                
+
                                                 if (_lexer.current() == lex_close_brace)
                                                 {
@@ -8585 +11407 @@
                                                         nt_type = parse_node_test_type(nt_name);
 
-                                                        if (nt_type == nodetest_none) throw_error("Unrecognized node type");
-                                                        
+                                                        if (nt_type == nodetest_none)
+                                                                throw_error("Unrecognized node type");
+
                                                         nt_name = xpath_lexer_string();
                                                 }
@@ -8593 +11416 @@
                                                         if (_lexer.current() != lex_quoted_string)
                                                                 throw_error("Only literals are allowed as arguments to processing-instruction()");
-                                                
+
                                                         nt_type = nodetest_pi;
                                                         nt_name = _lexer.contents();
                                                         _lexer.next();
-                                                        
+
                                                         if (_lexer.current() != lex_close_brace)
                                                                 throw_error("Unmatched brace near processing-instruction()");
@@ -8603 +11426 @@
                                                 }
                                                 else
+                                                {
                                                         throw_error("Unmatched brace near node type test");
-
+                                                }
                                         }
                                         // QName or NCName:*
@@ -8612 +11436 @@
                                                 {
                                                         nt_name.end--; // erase *
-                                                        
+
                                                         nt_type = nodetest_all_in_namespace;
                                                 }
-                                                else nt_type = nodetest_name;
+                                                else
+                                                {
+                                                        nt_type = nodetest_name;
+                                                }
                                         }
                                 }
@@ -8624 +11451 @@
                                 _lexer.next();
                         }
-                        else throw_error("Unrecognized node test");
-                        
-                        xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_step, set, axis, nt_type, alloc_string(nt_name));
-                        
+                        else
+                        {
+                                throw_error("Unrecognized node test");
+                        }
+
+                        const char_t* nt_name_copy = alloc_string(nt_name);
+                        xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_step, set, axis, nt_type, nt_name_copy);
+
                         xpath_ast_node* last = 0;
-                        
+
                         while (_lexer.current() == lex_open_square_brace)
                         {
                                 _lexer.next();
-                                
+
                                 xpath_ast_node* expr = parse_expression();
 
-                                xpath_ast_node* pred = new (alloc_node()) xpath_ast_node(ast_predicate, xpath_type_node_set, expr);
-                                
+                                xpath_ast_node* pred = new (alloc_node()) xpath_ast_node(ast_predicate, 0, expr, predicate_default);
+
                                 if (_lexer.current() != lex_close_square_brace)
-                                throw_error("Unmatched square brace");
+                                        throw_error("Unmatched square brace");
                                 _lexer.next();
-                                
+
                                 if (last) last->set_next(pred);
                                 else n->set_right(pred);
-                                
+
                                 last = pred;
                         }
-                        
+
                         return n;
-            }
-            
-            // RelativeLocationPath ::= Step | RelativeLocationPath '/' Step | RelativeLocationPath '//' Step
-            xpath_ast_node* parse_relative_location_path(xpath_ast_node* set)
-            {
+                }
+
+                // RelativeLocationPath ::= Step | RelativeLocationPath '/' Step | RelativeLocationPath '//' Step
+                xpath_ast_node* parse_relative_location_path(xpath_ast_node* set)
+                {
                         xpath_ast_node* n = parse_step(set);
-                        
+
                         while (_lexer.current() == lex_slash || _lexer.current() == lex_double_slash)
                         {
@@ -8663 +11494 @@
                                 if (l == lex_double_slash)
                                         n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);
-                                
+
                                 n = parse_step(n);
                         }
-                        
+
                         return n;
-            }
-            
-            // LocationPath ::= RelativeLocationPath | AbsoluteLocationPath
-            // AbsoluteLocationPath ::= '/' RelativeLocationPath? | '//' RelativeLocationPath
-            xpath_ast_node* parse_location_path()
-            {
+                }
+
+                // LocationPath ::= RelativeLocationPath | AbsoluteLocationPath
+                // AbsoluteLocationPath ::= '/' RelativeLocationPath? | '//' RelativeLocationPath
+                xpath_ast_node* parse_location_path()
+                {
                         if (_lexer.current() == lex_slash)
                         {
                                 _lexer.next();
-                                
+
                                 xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_step_root, xpath_type_node_set);
 
@@ -8691 +11522 @@
                         {
                                 _lexer.next();
-                                
+
                                 xpath_ast_node* n = new (alloc_node()) xpath_ast_node(ast_step_root, xpath_type_node_set);
                                 n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);
-                                
+
                                 return parse_relative_location_path(n);
                         }
@@ -8700 +11531 @@
                         // else clause moved outside of if because of bogus warning 'control may reach end of non-void function being inlined' in gcc 4.0.1
                         return parse_relative_location_path(0);
-            }
-            
-            // PathExpr ::= LocationPath
-            //                          | FilterExpr
-            //                          | FilterExpr '/' RelativeLocationPath
-            //                          | FilterExpr '//' RelativeLocationPath
-            xpath_ast_node* parse_path_expression()
-            {
+                }
+
+                // PathExpr ::= LocationPath
+                //                              | FilterExpr
+                //                              | FilterExpr '/' RelativeLocationPath
+                //                              | FilterExpr '//' RelativeLocationPath
+                // UnionExpr ::= PathExpr | UnionExpr '|' PathExpr
+                // UnaryExpr ::= UnionExpr | '-' UnaryExpr
+                xpath_ast_node* parse_path_or_unary_expression()
+                {
                         // Clarification.
                         // PathExpr begins with either LocationPath or FilterExpr.
@@ -8715 +11548 @@
                         // function call.
 
-                        if (_lexer.current() == lex_var_ref || _lexer.current() == lex_open_brace || 
+                        if (_lexer.current() == lex_var_ref || _lexer.current() == lex_open_brace ||
                                 _lexer.current() == lex_quoted_string || _lexer.current() == lex_number ||
                                 _lexer.current() == lex_string)
-                {
-                        if (_lexer.current() == lex_string)
-                        {
-                                // This is either a function call, or not - if not, we shall proceed with location path
-                                const char_t* state = _lexer.state();
-                                
-                                        while (IS_CHARTYPE(*state, ct_space)) ++state;
-                                
-                                if (*state != '(') return parse_location_path();
+                        {
+                                if (_lexer.current() == lex_string)
+                                {
+                                        // This is either a function call, or not - if not, we shall proceed with location path
+                                        const char_t* state = _lexer.state();
+
+                                        while (PUGI__IS_CHARTYPE(*state, ct_space)) ++state;
+
+                                        if (*state != '(') return parse_location_path();
 
                                         // This looks like a function call; however this still can be a node-test. Check it.
-                                        if (parse_node_test_type(_lexer.contents()) != nodetest_none) return parse_location_path();
-                        }
-                        
-                        xpath_ast_node* n = parse_filter_expression();
-
-                        if (_lexer.current() == lex_slash || _lexer.current() == lex_double_slash)
-                        {
+                                        if (parse_node_test_type(_lexer.contents()) != nodetest_none)
+                                                return parse_location_path();
+                                }
+
+                                xpath_ast_node* n = parse_filter_expression();
+
+                                if (_lexer.current() == lex_slash || _lexer.current() == lex_double_slash)
+                                {
                                         lexeme_t l = _lexer.current();
-                                _lexer.next();
-                                
+                                        _lexer.next();
+
                                         if (l == lex_double_slash)
                                         {
-                                                if (n->rettype() != xpath_type_node_set) throw_error("Step has to be applied to node set");
+                                                if (n->rettype() != xpath_type_node_set)
+                                                        throw_error("Step has to be applied to node set");
 
                                                 n = new (alloc_node()) xpath_ast_node(ast_step, n, axis_descendant_or_self, nodetest_type_node, 0);
                                         }
-        
-                                // select from location path
-                                return parse_relative_location_path(n);
-                        }
-
-                        return n;
-                }
-                else return parse_location_path();
-            }
-
-            // UnionExpr ::= PathExpr | UnionExpr '|' PathExpr
-            xpath_ast_node* parse_union_expression()
-            {
-                xpath_ast_node* n = parse_path_expression();
-
-                while (_lexer.current() == lex_union)
-                {
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_union_expression();
-
-                                if (n->rettype() != xpath_type_node_set || expr->rettype() != xpath_type_node_set)
+
+                                        // select from location path
+                                        return parse_relative_location_path(n);
+                                }
+
+                                return n;
+                        }
+                        else if (_lexer.current() == lex_minus)
+                        {
+                                _lexer.next();
+
+                                // precedence 7+ - only parses union expressions
+                                xpath_ast_node* expr = parse_expression_rec(parse_path_or_unary_expression(), 7);
+
+                                return new (alloc_node()) xpath_ast_node(ast_op_negate, xpath_type_number, expr);
+                        }
+                        else
+                        {
+                                return parse_location_path();
+                        }
+                }
+
+                struct binary_op_t
+                {
+                        ast_type_t asttype;
+                        xpath_value_type rettype;
+                        int precedence;
+
+                        binary_op_t(): asttype(ast_unknown), rettype(xpath_type_none), precedence(0)
+                        {
+                        }
+
+                        binary_op_t(ast_type_t asttype_, xpath_value_type rettype_, int precedence_): asttype(asttype_), rettype(rettype_), precedence(precedence_)
+                        {
+                        }
+
+                        static binary_op_t parse(xpath_lexer& lexer)
+                        {
+                                switch (lexer.current())
+                                {
+                                case lex_string:
+                                        if (lexer.contents() == PUGIXML_TEXT("or"))
+                                                return binary_op_t(ast_op_or, xpath_type_boolean, 1);
+                                        else if (lexer.contents() == PUGIXML_TEXT("and"))
+                                                return binary_op_t(ast_op_and, xpath_type_boolean, 2);
+                                        else if (lexer.contents() == PUGIXML_TEXT("div"))
+                                                return binary_op_t(ast_op_divide, xpath_type_number, 6);
+                                        else if (lexer.contents() == PUGIXML_TEXT("mod"))
+                                                return binary_op_t(ast_op_mod, xpath_type_number, 6);
+                                        else
+                                                return binary_op_t();
+
+                                case lex_equal:
+                                        return binary_op_t(ast_op_equal, xpath_type_boolean, 3);
+
+                                case lex_not_equal:
+                                        return binary_op_t(ast_op_not_equal, xpath_type_boolean, 3);
+
+                                case lex_less:
+                                        return binary_op_t(ast_op_less, xpath_type_boolean, 4);
+
+                                case lex_greater:
+                                        return binary_op_t(ast_op_greater, xpath_type_boolean, 4);
+
+                                case lex_less_or_equal:
+                                        return binary_op_t(ast_op_less_or_equal, xpath_type_boolean, 4);
+
+                                case lex_greater_or_equal:
+                                        return binary_op_t(ast_op_greater_or_equal, xpath_type_boolean, 4);
+
+                                case lex_plus:
+                                        return binary_op_t(ast_op_add, xpath_type_number, 5);
+
+                                case lex_minus:
+                                        return binary_op_t(ast_op_subtract, xpath_type_number, 5);
+
+                                case lex_multiply:
+                                        return binary_op_t(ast_op_multiply, xpath_type_number, 6);
+
+                                case lex_union:
+                                        return binary_op_t(ast_op_union, xpath_type_node_set, 7);
+
+                                default:
+                                        return binary_op_t();
+                                }
+                        }
+                };
+
+                xpath_ast_node* parse_expression_rec(xpath_ast_node* lhs, int limit)
+                {
+                        binary_op_t op = binary_op_t::parse(_lexer);
+
+                        while (op.asttype != ast_unknown && op.precedence >= limit)
+                        {
+                                _lexer.next();
+
+                                xpath_ast_node* rhs = parse_path_or_unary_expression();
+
+                                binary_op_t nextop = binary_op_t::parse(_lexer);
+
+                                while (nextop.asttype != ast_unknown && nextop.precedence > op.precedence)
+                                {
+                                        rhs = parse_expression_rec(rhs, nextop.precedence);
+
+                                        nextop = binary_op_t::parse(_lexer);
+                                }
+
+                                if (op.asttype == ast_op_union && (lhs->rettype() != xpath_type_node_set || rhs->rettype() != xpath_type_node_set))
                                         throw_error("Union operator has to be applied to node sets");
 
-                        n = new (alloc_node()) xpath_ast_node(ast_op_union, xpath_type_node_set, n, expr);
-                }
-
-                return n;
-            }
-
-            // UnaryExpr ::= UnionExpr | '-' UnaryExpr
-            xpath_ast_node* parse_unary_expression()
-            {
-                if (_lexer.current() == lex_minus)
-                {
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_unary_expression();
-
-                        return new (alloc_node()) xpath_ast_node(ast_op_negate, xpath_type_number, expr);
-                }
-                else return parse_union_expression();
-            }
-            
-            // MultiplicativeExpr ::= UnaryExpr
-            //                                            | MultiplicativeExpr '*' UnaryExpr
-            //                                            | MultiplicativeExpr 'div' UnaryExpr
-            //                                            | MultiplicativeExpr 'mod' UnaryExpr
-            xpath_ast_node* parse_multiplicative_expression()
-            {
-                xpath_ast_node* n = parse_unary_expression();
-
-                while (_lexer.current() == lex_multiply || (_lexer.current() == lex_string &&
-                           (_lexer.contents() == PUGIXML_TEXT("mod") || _lexer.contents() == PUGIXML_TEXT("div"))))
-                {
-                        ast_type_t op = _lexer.current() == lex_multiply ? ast_op_multiply :
-                                _lexer.contents().begin[0] == 'd' ? ast_op_divide : ast_op_mod;
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_unary_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(op, xpath_type_number, n, expr);
-                }
-
-                return n;
-            }
-
-            // AdditiveExpr ::= MultiplicativeExpr
-            //                                  | AdditiveExpr '+' MultiplicativeExpr
-            //                                  | AdditiveExpr '-' MultiplicativeExpr
-            xpath_ast_node* parse_additive_expression()
-            {
-                xpath_ast_node* n = parse_multiplicative_expression();
-
-                while (_lexer.current() == lex_plus || _lexer.current() == lex_minus)
-                {
-                        lexeme_t l = _lexer.current();
-
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_multiplicative_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(l == lex_plus ? ast_op_add : ast_op_subtract, xpath_type_number, n, expr);
-                }
-
-                return n;
-            }
-
-            // RelationalExpr ::= AdditiveExpr
-            //                                    | RelationalExpr '<' AdditiveExpr
-            //                                    | RelationalExpr '>' AdditiveExpr
-            //                                    | RelationalExpr '<=' AdditiveExpr
-            //                                    | RelationalExpr '>=' AdditiveExpr
-            xpath_ast_node* parse_relational_expression()
-            {
-                xpath_ast_node* n = parse_additive_expression();
-
-                while (_lexer.current() == lex_less || _lexer.current() == lex_less_or_equal || 
-                           _lexer.current() == lex_greater || _lexer.current() == lex_greater_or_equal)
-                {
-                        lexeme_t l = _lexer.current();
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_additive_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(l == lex_less ? ast_op_less : l == lex_greater ? ast_op_greater :
-                                                        l == lex_less_or_equal ? ast_op_less_or_equal : ast_op_greater_or_equal, xpath_type_boolean, n, expr);
-                }
-
-                return n;
-            }
-            
-            // EqualityExpr ::= RelationalExpr
-            //                                  | EqualityExpr '=' RelationalExpr
-            //                                  | EqualityExpr '!=' RelationalExpr
-            xpath_ast_node* parse_equality_expression()
-            {
-                xpath_ast_node* n = parse_relational_expression();
-
-                while (_lexer.current() == lex_equal || _lexer.current() == lex_not_equal)
-                {
-                        lexeme_t l = _lexer.current();
-
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_relational_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(l == lex_equal ? ast_op_equal : ast_op_not_equal, xpath_type_boolean, n, expr);
-                }
-
-                return n;
-            }
-            
-            // AndExpr ::= EqualityExpr | AndExpr 'and' EqualityExpr
-            xpath_ast_node* parse_and_expression()
-            {
-                xpath_ast_node* n = parse_equality_expression();
-
-                while (_lexer.current() == lex_string && _lexer.contents() == PUGIXML_TEXT("and"))
-                {
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_equality_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(ast_op_and, xpath_type_boolean, n, expr);
-                }
-
-                return n;
-            }
-
-            // OrExpr ::= AndExpr | OrExpr 'or' AndExpr
-            xpath_ast_node* parse_or_expression()
-            {
-                xpath_ast_node* n = parse_and_expression();
-
-                while (_lexer.current() == lex_string && _lexer.contents() == PUGIXML_TEXT("or"))
-                {
-                        _lexer.next();
-
-                                xpath_ast_node* expr = parse_and_expression();
-
-                        n = new (alloc_node()) xpath_ast_node(ast_op_or, xpath_type_boolean, n, expr);
-                }
-
-                return n;
-            }
-                
+                                lhs = new (alloc_node()) xpath_ast_node(op.asttype, op.rettype, lhs, rhs);
+
+                                op = binary_op_t::parse(_lexer);
+                        }
+
+                        return lhs;
+                }
+
                 // Expr ::= OrExpr
+                // OrExpr ::= AndExpr | OrExpr 'or' AndExpr
+                // AndExpr ::= EqualityExpr | AndExpr 'and' EqualityExpr
+                // EqualityExpr ::= RelationalExpr
+                //                                      | EqualityExpr '=' RelationalExpr
+                //                                      | EqualityExpr '!=' RelationalExpr
+                // RelationalExpr ::= AdditiveExpr
+                //                                        | RelationalExpr '<' AdditiveExpr
+                //                                        | RelationalExpr '>' AdditiveExpr
+                //                                        | RelationalExpr '<=' AdditiveExpr
+                //                                        | RelationalExpr '>=' AdditiveExpr
+                // AdditiveExpr ::= MultiplicativeExpr
+                //                                      | AdditiveExpr '+' MultiplicativeExpr
+                //                                      | AdditiveExpr '-' MultiplicativeExpr
+                // MultiplicativeExpr ::= UnaryExpr
+                //                                                | MultiplicativeExpr '*' UnaryExpr
+                //                                                | MultiplicativeExpr 'div' UnaryExpr
+                //                                                | MultiplicativeExpr 'mod' UnaryExpr
                 xpath_ast_node* parse_expression()
                 {
-                        return parse_or_expression();
+                        return parse_expression_rec(parse_path_or_unary_expression(), 0);
                 }
 
@@ -8925 +11728 @@
                 {
                         xpath_ast_node* result = parse_expression();
-                        
+
+                        // check if there are unparsed tokens left
                         if (_lexer.current() != lex_eof)
-                        {
-                                // there are still unparsed tokens left, error
                                 throw_error("Incorrect query");
-                        }
-                        
+
                         return result;
                 }
@@ -8949 +11750 @@
         };
 
-    struct xpath_query_impl
-    {
+        struct xpath_query_impl
+        {
                 static xpath_query_impl* create()
                 {
-                        void* memory = global_allocate(sizeof(xpath_query_impl));
-
-            return new (memory) xpath_query_impl();
-                }
-
-                static void destroy(void* ptr)
-                {
-                        if (!ptr) return;
-                        
+                        void* memory = xml_memory::allocate(sizeof(xpath_query_impl));
+                        if (!memory) return 0;
+
+                        return new (memory) xpath_query_impl();
+                }
+
+                static void destroy(xpath_query_impl* impl)
+                {
                         // free all allocated pages
-                        static_cast<xpath_query_impl*>(ptr)->alloc.release();
+                        impl->alloc.release();
 
                         // free allocator memory (with the first page)
-                        global_deallocate(ptr);
-                }
-
-        xpath_query_impl(): root(0), alloc(&block)
-        {
-            block.next = 0;
-        }
-
-        xpath_ast_node* root;
-        xpath_allocator alloc;
-        xpath_memory_block block;
-    };
-
-        xpath_string evaluate_string_impl(xpath_query_impl* impl, const xpath_node& n, xpath_stack_data& sd)
+                        xml_memory::deallocate(impl);
+                }
+
+                xpath_query_impl(): root(0), alloc(&block)
+                {
+                        block.next = 0;
+                        block.capacity = sizeof(block.data);
+                }
+
+                xpath_ast_node* root;
+                xpath_allocator alloc;
+                xpath_memory_block block;
+        };
+
+        PUGI__FN xpath_string evaluate_string_impl(xpath_query_impl* impl, const xpath_node& n, xpath_stack_data& sd)
         {
                 if (!impl) return xpath_string();
@@ -8991 +11792 @@
                 return impl->root->eval_string(c, sd.stack);
         }
-}
+
+        PUGI__FN impl::xpath_ast_node* evaluate_node_set_prepare(xpath_query_impl* impl)
+        {
+                if (!impl) return 0;
+
+                if (impl->root->rettype() != xpath_type_node_set)
+                {
+                #ifdef PUGIXML_NO_EXCEPTIONS
+                        return 0;
+                #else
+                        xpath_parse_result res;
+                        res.error = "Expression does not evaluate to node set";
+
+                        throw xpath_exception(res);
+                #endif
+                }
+
+                return impl->root;
+        }
+PUGI__NS_END
 
 namespace pugi
 {
 #ifndef PUGIXML_NO_EXCEPTIONS
-        xpath_exception::xpath_exception(const xpath_parse_result& result): _result(result)
-        {
-                assert(result.error);
-        }
-        
-        const char* xpath_exception::what() const throw()
+        PUGI__FN xpath_exception::xpath_exception(const xpath_parse_result& result_): _result(result_)
+        {
+                assert(_result.error);
+        }
+
+        PUGI__FN const char* xpath_exception::what() const throw()
         {
                 return _result.error;
         }
 
-        const xpath_parse_result& xpath_exception::result() const
+        PUGI__FN const xpath_parse_result& xpath_exception::result() const
         {
                 return _result;
         }
 #endif
-        
-        xpath_node::xpath_node()
-        {
-        }
-                
-        xpath_node::xpath_node(const xml_node& node): _node(node)
-        {
-        }
-                
-        xpath_node::xpath_node(const xml_attribute& attribute, const xml_node& parent): _node(attribute ? parent : xml_node()), _attribute(attribute)
-        {
-        }
-
-        xml_node xpath_node::node() const
+
+        PUGI__FN xpath_node::xpath_node()
+        {
+        }
+
+        PUGI__FN xpath_node::xpath_node(const xml_node& node_): _node(node_)
+        {
+        }
+
+        PUGI__FN xpath_node::xpath_node(const xml_attribute& attribute_, const xml_node& parent_): _node(attribute_ ? parent_ : xml_node()), _attribute(attribute_)
+        {
+        }
+
+        PUGI__FN xml_node xpath_node::node() const
         {
                 return _attribute ? xml_node() : _node;
         }
-                
-        xml_attribute xpath_node::attribute() const
+
+        PUGI__FN xml_attribute xpath_node::attribute() const
         {
                 return _attribute;
         }
-        
-        xml_node xpath_node::parent() const
+
+        PUGI__FN xml_node xpath_node::parent() const
         {
                 return _attribute ? _node : _node.parent();
         }
 
-        xpath_node::operator xpath_node::unspecified_bool_type() const
-        {
-                return (_node || _attribute) ? &xpath_node::_node : 0;
-        }
-        
-        bool xpath_node::operator!() const
+        PUGI__FN static void unspecified_bool_xpath_node(xpath_node***)
+        {
+        }
+
+        PUGI__FN xpath_node::operator xpath_node::unspecified_bool_type() const
+        {
+                return (_node || _attribute) ? unspecified_bool_xpath_node : 0;
+        }
+
+        PUGI__FN bool xpath_node::operator!() const
         {
                 return !(_node || _attribute);
         }
 
-        bool xpath_node::operator==(const xpath_node& n) const
+        PUGI__FN bool xpath_node::operator==(const xpath_node& n) const
         {
                 return _node == n._node && _attribute == n._attribute;
         }
-        
-        bool xpath_node::operator!=(const xpath_node& n) const
+
+        PUGI__FN bool xpath_node::operator!=(const xpath_node& n) const
         {
                 return _node != n._node || _attribute != n._attribute;
@@ -9060 +11884 @@
 
 #ifdef __BORLANDC__
-        bool operator&&(const xpath_node& lhs, bool rhs)
+        PUGI__FN bool operator&&(const xpath_node& lhs, bool rhs)
         {
                 return (bool)lhs && rhs;
         }
 
-        bool operator||(const xpath_node& lhs, bool rhs)
+        PUGI__FN bool operator||(const xpath_node& lhs, bool rhs)
         {
                 return (bool)lhs || rhs;
@@ -9071 +11895 @@
 #endif
 
-        void xpath_node_set::_assign(const_iterator begin, const_iterator end)
-        {
-                assert(begin <= end);
-
-                size_t size = static_cast<size_t>(end - begin);
-
-                if (size <= 1)
+        PUGI__FN void xpath_node_set::_assign(const_iterator begin_, const_iterator end_, type_t type_)
+        {
+                assert(begin_ <= end_);
+
+                size_t size_ = static_cast<size_t>(end_ - begin_);
+
+                if (size_ <= 1)
                 {
                         // deallocate old buffer
-                        if (_begin != &_storage) global_deallocate(_begin);
+                        if (_begin != &_storage) impl::xml_memory::deallocate(_begin);
 
                         // use internal buffer
-                        if (begin != end) _storage = *begin;
+                        if (begin_ != end_) _storage = *begin_;
 
                         _begin = &_storage;
-                        _end = &_storage + size;
+                        _end = &_storage + size_;
+                        _type = type_;
                 }
                 else
                 {
                         // make heap copy
-                        xpath_node* storage = static_cast<xpath_node*>(global_allocate(size * sizeof(xpath_node)));
+                        xpath_node* storage = static_cast<xpath_node*>(impl::xml_memory::allocate(size_ * sizeof(xpath_node)));
 
                         if (!storage)
@@ -9102 +11927 @@
                         }
 
-                        memcpy(storage, begin, size * sizeof(xpath_node));
-                        
+                        memcpy(storage, begin_, size_ * sizeof(xpath_node));
+
                         // deallocate old buffer
-                        if (_begin != &_storage) global_deallocate(_begin);
+                        if (_begin != &_storage) impl::xml_memory::deallocate(_begin);
 
                         // finalize
                         _begin = storage;
-                        _end = storage + size;
-                }
-        }
-
-        xpath_node_set::xpath_node_set(): _type(type_unsorted), _begin(&_storage), _end(&_storage)
-        {
-        }
-
-        xpath_node_set::xpath_node_set(const_iterator begin, const_iterator end, type_t type): _type(type), _begin(&_storage), _end(&_storage)
-        {
-                _assign(begin, end);
-        }
-
-        xpath_node_set::~xpath_node_set()
-        {
-                if (_begin != &_storage) global_deallocate(_begin);
-        }
-                
-        xpath_node_set::xpath_node_set(const xpath_node_set& ns): _type(ns._type), _begin(&_storage), _end(&_storage)
-        {
-                _assign(ns._begin, ns._end);
-        }
-        
-        xpath_node_set& xpath_node_set::operator=(const xpath_node_set& ns)
+                        _end = storage + size_;
+                        _type = type_;
+                }
+        }
+
+#ifdef PUGIXML_HAS_MOVE
+        PUGI__FN void xpath_node_set::_move(xpath_node_set& rhs)
+        {
+                _type = rhs._type;
+                _storage = rhs._storage;
+                _begin = (rhs._begin == &rhs._storage) ? &_storage : rhs._begin;
+                _end = _begin + (rhs._end - rhs._begin);
+
+                rhs._type = type_unsorted;
+                rhs._begin = &rhs._storage;
+                rhs._end = rhs._begin;
+        }
+#endif
+
+        PUGI__FN xpath_node_set::xpath_node_set(): _type(type_unsorted), _begin(&_storage), _end(&_storage)
+        {
+        }
+
+        PUGI__FN xpath_node_set::xpath_node_set(const_iterator begin_, const_iterator end_, type_t type_): _type(type_unsorted), _begin(&_storage), _end(&_storage)
+        {
+                _assign(begin_, end_, type_);
+        }
+
+        PUGI__FN xpath_node_set::~xpath_node_set()
+        {
+                if (_begin != &_storage)
+                        impl::xml_memory::deallocate(_begin);
+        }
+
+        PUGI__FN xpath_node_set::xpath_node_set(const xpath_node_set& ns): _type(type_unsorted), _begin(&_storage), _end(&_storage)
+        {
+                _assign(ns._begin, ns._end, ns._type);
+        }
+
+        PUGI__FN xpath_node_set& xpath_node_set::operator=(const xpath_node_set& ns)
         {
                 if (this == &ns) return *this;
-                
-                _type = ns._type;
-                _assign(ns._begin, ns._end);
+
+                _assign(ns._begin, ns._end, ns._type);
 
                 return *this;
         }
 
-        xpath_node_set::type_t xpath_node_set::type() const
+#ifdef PUGIXML_HAS_MOVE
+        PUGI__FN xpath_node_set::xpath_node_set(xpath_node_set&& rhs): _type(type_unsorted), _begin(&_storage), _end(&_storage)
+        {
+                _move(rhs);
+        }
+
+        PUGI__FN xpath_node_set& xpath_node_set::operator=(xpath_node_set&& rhs)
+        {
+                if (this == &rhs) return *this;
+
+                if (_begin != &_storage)
+                        impl::xml_memory::deallocate(_begin);
+
+                _move(rhs);
+
+                return *this;
+        }
+#endif
+
+        PUGI__FN xpath_node_set::type_t xpath_node_set::type() const
         {
                 return _type;
         }
-                
-        size_t xpath_node_set::size() const
+
+        PUGI__FN size_t xpath_node_set::size() const
         {
                 return _end - _begin;
         }
-                
-        bool xpath_node_set::empty() const
+
+        PUGI__FN bool xpath_node_set::empty() const
         {
                 return _begin == _end;
         }
-                
-        const xpath_node& xpath_node_set::operator[](size_t index) const
+
+        PUGI__FN const xpath_node& xpath_node_set::operator[](size_t index) const
         {
                 assert(index < size());
@@ -9163 +12022 @@
         }
 
-        xpath_node_set::const_iterator xpath_node_set::begin() const
+        PUGI__FN xpath_node_set::const_iterator xpath_node_set::begin() const
         {
                 return _begin;
         }
-                
-        xpath_node_set::const_iterator xpath_node_set::end() const
+
+        PUGI__FN xpath_node_set::const_iterator xpath_node_set::end() const
         {
                 return _end;
         }
-        
-        void xpath_node_set::sort(bool reverse)
-        {
-                _type = xpath_sort(_begin, _end, _type, reverse);
-        }
-
-        xpath_node xpath_node_set::first() const
-        {
-                return xpath_first(_begin, _end, _type);
-        }
-
-    xpath_parse_result::xpath_parse_result(): error("Internal error"), offset(0)
-    {
-    }
-
-    xpath_parse_result::operator bool() const
-    {
-        return error == 0;
-    }
-        const char* xpath_parse_result::description() const
+
+        PUGI__FN void xpath_node_set::sort(bool reverse)
+        {
+                _type = impl::xpath_sort(_begin, _end, _type, reverse);
+        }
+
+        PUGI__FN xpath_node xpath_node_set::first() const
+        {
+                return impl::xpath_first(_begin, _end, _type);
+        }
+
+        PUGI__FN xpath_parse_result::xpath_parse_result(): error("Internal error"), offset(0)
+        {
+        }
+
+        PUGI__FN xpath_parse_result::operator bool() const
+        {
+                return error == 0;
+        }
+
+        PUGI__FN const char* xpath_parse_result::description() const
         {
                 return error ? error : "No error";
         }
 
-        xpath_variable::xpath_variable()
-    {
-    }
-
-        const char_t* xpath_variable::name() const
+        PUGI__FN xpath_variable::xpath_variable(xpath_value_type type_): _type(type_), _next(0)
+        {
+        }
+
+        PUGI__FN const char_t* xpath_variable::name() const
         {
                 switch (_type)
                 {
                 case xpath_type_node_set:
-                        return static_cast<const xpath_variable_node_set*>(this)->name;
+                        return static_cast<const impl::xpath_variable_node_set*>(this)->name;
 
                 case xpath_type_number:
-                        return static_cast<const xpath_variable_number*>(this)->name;
+                        return static_cast<const impl::xpath_variable_number*>(this)->name;
 
                 case xpath_type_string:
-                        return static_cast<const xpath_variable_string*>(this)->name;
+                        return static_cast<const impl::xpath_variable_string*>(this)->name;
 
                 case xpath_type_boolean:
-                        return static_cast<const xpath_variable_boolean*>(this)->name;
+                        return static_cast<const impl::xpath_variable_boolean*>(this)->name;
 
                 default:
-                        assert(!"Invalid variable type");
+                        assert(false && "Invalid variable type");
                         return 0;
                 }
         }
 
-        xpath_value_type xpath_variable::type() const
+        PUGI__FN xpath_value_type xpath_variable::type() const
         {
                 return _type;
         }
 
-        bool xpath_variable::get_boolean() const
-        {
-                return (_type == xpath_type_boolean) ? static_cast<const xpath_variable_boolean*>(this)->value : false;
-        }
-
-        double xpath_variable::get_number() const
-        {
-                return (_type == xpath_type_number) ? static_cast<const xpath_variable_number*>(this)->value : gen_nan();
-        }
-
-        const char_t* xpath_variable::get_string() const
-        {
-                const char_t* value = (_type == xpath_type_string) ? static_cast<const xpath_variable_string*>(this)->value : 0;
+        PUGI__FN bool xpath_variable::get_boolean() const
+        {
+                return (_type == xpath_type_boolean) ? static_cast<const impl::xpath_variable_boolean*>(this)->value : false;
+        }
+
+        PUGI__FN double xpath_variable::get_number() const
+        {
+                return (_type == xpath_type_number) ? static_cast<const impl::xpath_variable_number*>(this)->value : impl::gen_nan();
+        }
+
+        PUGI__FN const char_t* xpath_variable::get_string() const
+        {
+                const char_t* value = (_type == xpath_type_string) ? static_cast<const impl::xpath_variable_string*>(this)->value : 0;
                 return value ? value : PUGIXML_TEXT("");
         }
 
-        const xpath_node_set& xpath_variable::get_node_set() const
-        {
-                return (_type == xpath_type_node_set) ? static_cast<const xpath_variable_node_set*>(this)->value : dummy_node_set;
-        }
-
-        bool xpath_variable::set(bool value)
+        PUGI__FN const xpath_node_set& xpath_variable::get_node_set() const
+        {
+                return (_type == xpath_type_node_set) ? static_cast<const impl::xpath_variable_node_set*>(this)->value : impl::dummy_node_set;
+        }
+
+        PUGI__FN bool xpath_variable::set(bool value)
         {
                 if (_type != xpath_type_boolean) return false;
 
-                static_cast<xpath_variable_boolean*>(this)->value = value;
+                static_cast<impl::xpath_variable_boolean*>(this)->value = value;
                 return true;
         }
 
-        bool xpath_variable::set(double value)
+        PUGI__FN bool xpath_variable::set(double value)
         {
                 if (_type != xpath_type_number) return false;
 
-                static_cast<xpath_variable_number*>(this)->value = value;
+                static_cast<impl::xpath_variable_number*>(this)->value = value;
                 return true;
         }
 
-        bool xpath_variable::set(const char_t* value)
+        PUGI__FN bool xpath_variable::set(const char_t* value)
         {
                 if (_type != xpath_type_string) return false;
 
-                xpath_variable_string* var = static_cast<xpath_variable_string*>(this);
+                impl::xpath_variable_string* var = static_cast<impl::xpath_variable_string*>(this);
 
                 // duplicate string
-                size_t size = (strlength(value) + 1) * sizeof(char_t);
-
-                char_t* copy = static_cast<char_t*>(global_allocate(size));
+                size_t size = (impl::strlength(value) + 1) * sizeof(char_t);
+
+                char_t* copy = static_cast<char_t*>(impl::xml_memory::allocate(size));
                 if (!copy) return false;
 
@@ -9279 +12139 @@
 
                 // replace old string
-                if (var->value) global_deallocate(var->value);
+                if (var->value) impl::xml_memory::deallocate(var->value);
                 var->value = copy;
 
@@ -9285 +12145 @@
         }
 
-        bool xpath_variable::set(const xpath_node_set& value)
+        PUGI__FN bool xpath_variable::set(const xpath_node_set& value)
         {
                 if (_type != xpath_type_node_set) return false;
 
-                static_cast<xpath_variable_node_set*>(this)->value = value;
+                static_cast<impl::xpath_variable_node_set*>(this)->value = value;
                 return true;
         }
 
-        xpath_variable_set::xpath_variable_set()
-        {
-                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i) _data[i] = 0;
-        }
-
-        xpath_variable_set::~xpath_variable_set()
+        PUGI__FN xpath_variable_set::xpath_variable_set()
         {
                 for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
-                {
-                        xpath_variable* var = _data[i];
-
-                        while (var)
-                        {
-                                xpath_variable* next = var->_next;
-
-                                delete_xpath_variable(var->_type, var);
-
-                                var = next;
-                        }
-                }
-        }
-
-        xpath_variable* xpath_variable_set::find(const char_t* name) const
+                        _data[i] = 0;
+        }
+
+        PUGI__FN xpath_variable_set::~xpath_variable_set()
+        {
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                        _destroy(_data[i]);
+        }
+
+        PUGI__FN xpath_variable_set::xpath_variable_set(const xpath_variable_set& rhs)
+        {
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                        _data[i] = 0;
+
+                _assign(rhs);
+        }
+
+        PUGI__FN xpath_variable_set& xpath_variable_set::operator=(const xpath_variable_set& rhs)
+        {
+                if (this == &rhs) return *this;
+
+                _assign(rhs);
+
+                return *this;
+        }
+
+#ifdef PUGIXML_HAS_MOVE
+        PUGI__FN xpath_variable_set::xpath_variable_set(xpath_variable_set&& rhs)
+        {
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                {
+                        _data[i] = rhs._data[i];
+                        rhs._data[i] = 0;
+                }
+        }
+
+        PUGI__FN xpath_variable_set& xpath_variable_set::operator=(xpath_variable_set&& rhs)
+        {
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                {
+                        _destroy(_data[i]);
+
+                        _data[i] = rhs._data[i];
+                        rhs._data[i] = 0;
+                }
+
+                return *this;
+        }
+#endif
+
+        PUGI__FN void xpath_variable_set::_assign(const xpath_variable_set& rhs)
+        {
+                xpath_variable_set temp;
+
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                        if (rhs._data[i] && !_clone(rhs._data[i], &temp._data[i]))
+                                return;
+
+                _swap(temp);
+        }
+
+        PUGI__FN void xpath_variable_set::_swap(xpath_variable_set& rhs)
+        {
+                for (size_t i = 0; i < sizeof(_data) / sizeof(_data[0]); ++i)
+                {
+                        xpath_variable* chain = _data[i];
+
+                        _data[i] = rhs._data[i];
+                        rhs._data[i] = chain;
+                }
+        }
+
+        PUGI__FN xpath_variable* xpath_variable_set::_find(const char_t* name) const
         {
                 const size_t hash_size = sizeof(_data) / sizeof(_data[0]);
-                size_t hash = hash_string(name) % hash_size;
+                size_t hash = impl::hash_string(name) % hash_size;
 
                 // look for existing variable
                 for (xpath_variable* var = _data[hash]; var; var = var->_next)
-                        if (strequal(var->name(), name))
+                        if (impl::strequal(var->name(), name))
                                 return var;
 
@@ -9328 +12241 @@
         }
 
-        xpath_variable* xpath_variable_set::add(const char_t* name, xpath_value_type type)
+        PUGI__FN bool xpath_variable_set::_clone(xpath_variable* var, xpath_variable** out_result)
+        {
+                xpath_variable* last = 0;
+
+                while (var)
+                {
+                        // allocate storage for new variable
+                        xpath_variable* nvar = impl::new_xpath_variable(var->_type, var->name());
+                        if (!nvar) return false;
+
+                        // link the variable to the result immediately to handle failures gracefully
+                        if (last)
+                                last->_next = nvar;
+                        else
+                                *out_result = nvar;
+
+                        last = nvar;
+
+                        // copy the value; this can fail due to out-of-memory conditions
+                        if (!impl::copy_xpath_variable(nvar, var)) return false;
+
+                        var = var->_next;
+                }
+
+                return true;
+        }
+
+        PUGI__FN void xpath_variable_set::_destroy(xpath_variable* var)
+        {
+                while (var)
+                {
+                        xpath_variable* next = var->_next;
+
+                        impl::delete_xpath_variable(var->_type, var);
+
+                        var = next;
+                }
+        }
+
+        PUGI__FN xpath_variable* xpath_variable_set::add(const char_t* name, xpath_value_type type)
         {
                 const size_t hash_size = sizeof(_data) / sizeof(_data[0]);
-                size_t hash = hash_string(name) % hash_size;
+                size_t hash = impl::hash_string(name) % hash_size;
 
                 // look for existing variable
                 for (xpath_variable* var = _data[hash]; var; var = var->_next)
-                        if (strequal(var->name(), name))
+                        if (impl::strequal(var->name(), name))
                                 return var->type() == type ? var : 0;
 
                 // add new variable
-                xpath_variable* result = new_xpath_variable(type, name);
+                xpath_variable* result = impl::new_xpath_variable(type, name);
 
                 if (result)
                 {
-                        result->_type = type;
                         result->_next = _data[hash];
 
@@ -9352 +12303 @@
         }
 
-        bool xpath_variable_set::set(const char_t* name, bool value)
+        PUGI__FN bool xpath_variable_set::set(const char_t* name, bool value)
         {
                 xpath_variable* var = add(name, xpath_type_boolean);
@@ -9358 +12309 @@
         }
 
-        bool xpath_variable_set::set(const char_t* name, double value)
+        PUGI__FN bool xpath_variable_set::set(const char_t* name, double value)
         {
                 xpath_variable* var = add(name, xpath_type_number);
@@ -9364 +12315 @@
         }
 
-        bool xpath_variable_set::set(const char_t* name, const char_t* value)
+        PUGI__FN bool xpath_variable_set::set(const char_t* name, const char_t* value)
         {
                 xpath_variable* var = add(name, xpath_type_string);
@@ -9370 +12321 @@
         }
 
-        bool xpath_variable_set::set(const char_t* name, const xpath_node_set& value)
+        PUGI__FN bool xpath_variable_set::set(const char_t* name, const xpath_node_set& value)
         {
                 xpath_variable* var = add(name, xpath_type_node_set);
@@ -9376 +12327 @@
         }
 
-        xpath_variable* xpath_variable_set::get(const char_t* name)
-        {
-                return find(name);
-        }
-
-        const xpath_variable* xpath_variable_set::get(const char_t* name) const
-        {
-                return find(name);
-        }
-
-        xpath_query::xpath_query(const char_t* query, xpath_variable_set* variables): _impl(0)
-        {
-                xpath_query_impl* impl = xpath_query_impl::create();
-
-                if (!impl)
+        PUGI__FN xpath_variable* xpath_variable_set::get(const char_t* name)
+        {
+                return _find(name);
+        }
+
+        PUGI__FN const xpath_variable* xpath_variable_set::get(const char_t* name) const
+        {
+                return _find(name);
+        }
+
+        PUGI__FN xpath_query::xpath_query(const char_t* query, xpath_variable_set* variables): _impl(0)
+        {
+                impl::xpath_query_impl* qimpl = impl::xpath_query_impl::create();
+
+                if (!qimpl)
                 {
                 #ifdef PUGIXML_NO_EXCEPTIONS
                         _result.error = "Out of memory";
-        #else
+                #else
                         throw std::bad_alloc();
                 #endif
@@ -9400 +12351 @@
                 else
                 {
-                        buffer_holder impl_holder(impl, xpath_query_impl::destroy);
-
-                        impl->root = xpath_parser::parse(query, variables, &impl->alloc, &_result);
-
-                        if (impl->root)
-                        {
-                _impl = static_cast<xpath_query_impl*>(impl_holder.release());
+                        using impl::auto_deleter; // MSVC7 workaround
+                        auto_deleter<impl::xpath_query_impl> impl(qimpl, impl::xpath_query_impl::destroy);
+
+                        qimpl->root = impl::xpath_parser::parse(query, variables, &qimpl->alloc, &_result);
+
+                        if (qimpl->root)
+                        {
+                                qimpl->root->optimize(&qimpl->alloc);
+
+                                _impl = impl.release();
                                 _result.error = 0;
                         }
@@ -9412 +12366 @@
         }
 
-        xpath_query::~xpath_query()
-        {
-                xpath_query_impl::destroy(_impl);
-        }
-
-        xpath_value_type xpath_query::return_type() const
+        PUGI__FN xpath_query::xpath_query(): _impl(0)
+        {
+        }
+
+        PUGI__FN xpath_query::~xpath_query()
+        {
+                if (_impl)
+                        impl::xpath_query_impl::destroy(static_cast<impl::xpath_query_impl*>(_impl));
+        }
+
+#ifdef PUGIXML_HAS_MOVE
+        PUGI__FN xpath_query::xpath_query(xpath_query&& rhs)
+        {
+                _impl = rhs._impl;
+                _result = rhs._result;
+                rhs._impl = 0;
+                rhs._result = xpath_parse_result();
+        }
+
+        PUGI__FN xpath_query& xpath_query::operator=(xpath_query&& rhs)
+        {
+                if (this == &rhs) return *this;
+
+                if (_impl)
+                        impl::xpath_query_impl::destroy(static_cast<impl::xpath_query_impl*>(_impl));
+
+                _impl = rhs._impl;
+                _result = rhs._result;
+                rhs._impl = 0;
+                rhs._result = xpath_parse_result();
+
+                return *this;
+        }
+#endif
+
+        PUGI__FN xpath_value_type xpath_query::return_type() const
         {
                 if (!_impl) return xpath_type_none;
 
-                return static_cast<xpath_query_impl*>(_impl)->root->rettype();
-        }
-
-        bool xpath_query::evaluate_boolean(const xpath_node& n) const
+                return static_cast<impl::xpath_query_impl*>(_impl)->root->rettype();
+        }
+
+        PUGI__FN bool xpath_query::evaluate_boolean(const xpath_node& n) const
         {
                 if (!_impl) return false;
-                
-                xpath_context c(n, 1, 1);
-                xpath_stack_data sd;
+
+                impl::xpath_context c(n, 1, 1);
+                impl::xpath_stack_data sd;
 
         #ifdef PUGIXML_NO_EXCEPTIONS
                 if (setjmp(sd.error_handler)) return false;
         #endif
-                
-                return static_cast<xpath_query_impl*>(_impl)->root->eval_boolean(c, sd.stack);
-        }
-        
-        double xpath_query::evaluate_number(const xpath_node& n) const
-        {
-                if (!_impl) return gen_nan();
-                
-                xpath_context c(n, 1, 1);
-                xpath_stack_data sd;
+
+                return static_cast<impl::xpath_query_impl*>(_impl)->root->eval_boolean(c, sd.stack);
+        }
+
+        PUGI__FN double xpath_query::evaluate_number(const xpath_node& n) const
+        {
+                if (!_impl) return impl::gen_nan();
+
+                impl::xpath_context c(n, 1, 1);
+                impl::xpath_stack_data sd;
 
         #ifdef PUGIXML_NO_EXCEPTIONS
-                if (setjmp(sd.error_handler)) return gen_nan();
+                if (setjmp(sd.error_handler)) return impl::gen_nan();
         #endif
 
-                return static_cast<xpath_query_impl*>(_impl)->root->eval_number(c, sd.stack);
+                return static_cast<impl::xpath_query_impl*>(_impl)->root->eval_number(c, sd.stack);
         }
 
 #ifndef PUGIXML_NO_STL
-        string_t xpath_query::evaluate_string(const xpath_node& n) const
-        {
-                xpath_stack_data sd;
-
-                return evaluate_string_impl(static_cast<xpath_query_impl*>(_impl), n, sd).c_str();
+        PUGI__FN string_t xpath_query::evaluate_string(const xpath_node& n) const
+        {
+                impl::xpath_stack_data sd;
+
+                impl::xpath_string r = impl::evaluate_string_impl(static_cast<impl::xpath_query_impl*>(_impl), n, sd);
+
+                return string_t(r.c_str(), r.length());
         }
 #endif
 
-        size_t xpath_query::evaluate_string(char_t* buffer, size_t capacity, const xpath_node& n) const
-        {
-                xpath_stack_data sd;
-
-                xpath_string r = evaluate_string_impl(static_cast<xpath_query_impl*>(_impl), n, sd);
+        PUGI__FN size_t xpath_query::evaluate_string(char_t* buffer, size_t capacity, const xpath_node& n) const
+        {
+                impl::xpath_stack_data sd;
+
+                impl::xpath_string r = impl::evaluate_string_impl(static_cast<impl::xpath_query_impl*>(_impl), n, sd);
 
                 size_t full_size = r.length() + 1;
-                
+
                 if (capacity > 0)
-        {
-            size_t size = (full_size < capacity) ? full_size : capacity;
-            assert(size > 0);
-
-            memcpy(buffer, r.c_str(), (size - 1) * sizeof(char_t));
-            buffer[size - 1] = 0;
-        }
-                
+                {
+                        size_t size = (full_size < capacity) ? full_size : capacity;
+                        assert(size > 0);
+
+                        memcpy(buffer, r.c_str(), (size - 1) * sizeof(char_t));
+                        buffer[size - 1] = 0;
+                }
+
                 return full_size;
         }
 
-        xpath_node_set xpath_query::evaluate_node_set(const xpath_node& n) const
-        {
-                if (!_impl) return xpath_node_set();
-
-        xpath_ast_node* root = static_cast<xpath_query_impl*>(_impl)->root;
-
-                if (root->rettype() != xpath_type_node_set)
-                {
-                #ifdef PUGIXML_NO_EXCEPTIONS
-                        return xpath_node_set();
-                #else
-                        xpath_parse_result result;
-                        result.error = "Expression does not evaluate to node set";
-
-                        throw xpath_exception(result);
-                #endif
-                }
-                
-                xpath_context c(n, 1, 1);
-                xpath_stack_data sd;
+        PUGI__FN xpath_node_set xpath_query::evaluate_node_set(const xpath_node& n) const
+        {
+                impl::xpath_ast_node* root = impl::evaluate_node_set_prepare(static_cast<impl::xpath_query_impl*>(_impl));
+                if (!root) return xpath_node_set();
+
+                impl::xpath_context c(n, 1, 1);
+                impl::xpath_stack_data sd;
 
         #ifdef PUGIXML_NO_EXCEPTIONS
@@ -9506 +12479 @@
         #endif
 
-                xpath_node_set_raw r = root->eval_node_set(c, sd.stack);
+                impl::xpath_node_set_raw r = root->eval_node_set(c, sd.stack, impl::nodeset_eval_all);
 
                 return xpath_node_set(r.begin(), r.end(), r.type());
         }
 
-        const xpath_parse_result& xpath_query::result() const
+        PUGI__FN xpath_node xpath_query::evaluate_node(const xpath_node& n) const
+        {
+                impl::xpath_ast_node* root = impl::evaluate_node_set_prepare(static_cast<impl::xpath_query_impl*>(_impl));
+                if (!root) return xpath_node();
+
+                impl::xpath_context c(n, 1, 1);
+                impl::xpath_stack_data sd;
+
+        #ifdef PUGIXML_NO_EXCEPTIONS
+                if (setjmp(sd.error_handler)) return xpath_node();
+        #endif
+
+                impl::xpath_node_set_raw r = root->eval_node_set(c, sd.stack, impl::nodeset_eval_first);
+
+                return r.first();
+        }
+
+        PUGI__FN const xpath_parse_result& xpath_query::result() const
         {
                 return _result;
         }
 
-        xpath_query::operator xpath_query::unspecified_bool_type() const
-        {
-                return _impl ? &xpath_query::_impl : 0;
-        }
-
-        bool xpath_query::operator!() const
+        PUGI__FN static void unspecified_bool_xpath_query(xpath_query***)
+        {
+        }
+
+        PUGI__FN xpath_query::operator xpath_query::unspecified_bool_type() const
+        {
+                return _impl ? unspecified_bool_xpath_query : 0;
+        }
+
+        PUGI__FN bool xpath_query::operator!() const
         {
                 return !_impl;
         }
 
-        xpath_node xml_node::select_single_node(const char_t* query, xpath_variable_set* variables) const
+        PUGI__FN xpath_node xml_node::select_node(const char_t* query, xpath_variable_set* variables) const
+        {
+                xpath_query q(query, variables);
+                return select_node(q);
+        }
+
+        PUGI__FN xpath_node xml_node::select_node(const xpath_query& query) const
+        {
+                return query.evaluate_node(*this);
+        }
+
+        PUGI__FN xpath_node_set xml_node::select_nodes(const char_t* query, xpath_variable_set* variables) const
+        {
+                xpath_query q(query, variables);
+                return select_nodes(q);
+        }
+
+        PUGI__FN xpath_node_set xml_node::select_nodes(const xpath_query& query) const
+        {
+                return query.evaluate_node_set(*this);
+        }
+
+        PUGI__FN xpath_node xml_node::select_single_node(const char_t* query, xpath_variable_set* variables) const
         {
                 xpath_query q(query, variables);
@@ -9532 +12548 @@
         }
 
-        xpath_node xml_node::select_single_node(const xpath_query& query) const
-        {
-                xpath_node_set s = query.evaluate_node_set(*this);
-                return s.empty() ? xpath_node() : s.first();
-        }
-
-        xpath_node_set xml_node::select_nodes(const char_t* query, xpath_variable_set* variables) const
-        {
-                xpath_query q(query, variables);
-                return select_nodes(q);
-        }
-
-        xpath_node_set xml_node::select_nodes(const xpath_query& query) const
-        {
-                return query.evaluate_node_set(*this);
+        PUGI__FN xpath_node xml_node::select_single_node(const xpath_query& query) const
+        {
+                return query.evaluate_node(*this);
         }
 }
@@ -9552 +12556 @@
 #endif
 
+#ifdef __BORLANDC__
+#       pragma option pop
+#endif
+
+// Intel C++ does not properly keep warning state for function templates,
+// so popping warning state at the end of translation unit leads to warnings in the middle.
+#if defined(_MSC_VER) && !defined(__INTEL_COMPILER)
+#       pragma warning(pop)
+#endif
+
+// Undefine all local macros (makes sure we're not leaking macros in header-only mode)
+#undef PUGI__NO_INLINE
+#undef PUGI__UNLIKELY
+#undef PUGI__STATIC_ASSERT
+#undef PUGI__DMC_VOLATILE
+#undef PUGI__MSVC_CRT_VERSION
+#undef PUGI__NS_BEGIN
+#undef PUGI__NS_END
+#undef PUGI__FN
+#undef PUGI__FN_NO_INLINE
+#undef PUGI__GETHEADER_IMPL
+#undef PUGI__GETPAGE_IMPL
+#undef PUGI__GETPAGE
+#undef PUGI__NODETYPE
+#undef PUGI__IS_CHARTYPE_IMPL
+#undef PUGI__IS_CHARTYPE
+#undef PUGI__IS_CHARTYPEX
+#undef PUGI__ENDSWITH
+#undef PUGI__SKIPWS
+#undef PUGI__OPTSET
+#undef PUGI__PUSHNODE
+#undef PUGI__POPNODE
+#undef PUGI__SCANFOR
+#undef PUGI__SCANWHILE
+#undef PUGI__SCANWHILE_UNROLL
+#undef PUGI__ENDSEG
+#undef PUGI__THROW_ERROR
+#undef PUGI__CHECK_ERROR
+
+#endif
+
 /**
- * Copyright (c) 2006-2010 Arseny Kapoulkine
+ * Copyright (c) 2006-2016 Arseny Kapoulkine
  *
  * Permission is hereby granted, free of charge, to any person
@@ -9566 +12611 @@
  * The above copyright notice and this permission notice shall be
  * included in all copies or substantial portions of the Software.
- * 
+ *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES